diff --git a/be/src/core/column/column_spatial.cpp b/be/src/core/column/column_spatial.cpp new file mode 100644 index 00000000000000..1e3fd7c624b2b5 --- /dev/null +++ b/be/src/core/column/column_spatial.cpp @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/column_spatial.h" + +#include +#include + +#include "common/exception.h" +#include "core/column/columns_common.h" +#include "exec/sort/sort_block.h" + +namespace doris { + +void ColumnSpatial::insert_data(const char* pos, size_t length) { + if (length <= StringView::kInlineSize) { + _data.emplace_back(pos, cast_set(length)); + } else { + _data.emplace_back(_arena.insert(pos, length), cast_set(length)); + } +} + +int ColumnSpatial::compare_at(size_t n, size_t m, const IColumn& rhs, + int /* nan_direction_hint */) const { + const auto& spatial = assert_cast(rhs); + DCHECK_EQ(_primitive_type, spatial._primitive_type); + return _data[n].compare(spatial._data[m]); +} + +MutableColumnPtr ColumnSpatial::clone_resized(size_t size) const { + auto result = create(_primitive_type); + const size_t copied = std::min(this->size(), size); + for (size_t i = 0; i < copied; ++i) { + const auto value = get_data_at(i); + result->insert_data(value.data, value.size); + } + result->insert_many_defaults(size - copied); + return result; +} + +void ColumnSpatial::insert_range_from(const IColumn& src, size_t start, size_t length) { + const auto& spatial = assert_cast(src); + DCHECK_EQ(_primitive_type, spatial._primitive_type); + if (start + length > spatial.size()) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "Spatial column range start = {}, length = {} is out of bounds for {} rows", + start, length, spatial.size()); + } + for (size_t i = 0; i < length; ++i) { + const auto value = spatial.get_data_at(start + i); + insert_data(value.data, value.size); + } +} + +void ColumnSpatial::insert_indices_from(const IColumn& src, const uint32_t* begin, + const uint32_t* end) { + const auto& spatial = assert_cast(src); + DCHECK_EQ(_primitive_type, spatial._primitive_type); + for (auto it = begin; it != end; ++it) { + const auto value = spatial.get_data_at(*it); + insert_data(value.data, value.size); + } +} + +bool ColumnSpatial::has_enough_capacity(const IColumn& src) const { + const auto& spatial = assert_cast(src); + return _data.capacity() - _data.size() > spatial.size(); +} + +ColumnPtr ColumnSpatial::filter(const IColumn::Filter& filter, ssize_t result_size_hint) const { + column_match_filter_size(size(), filter.size()); + auto result = create(_primitive_type); + if (result_size_hint > 0) { + result->_data.reserve(result_size_hint); + } + for (size_t i = 0; i < size(); ++i) { + if (filter[i]) { + const auto value = get_data_at(i); + result->insert_data(value.data, value.size); + } + } + return result; +} + +size_t ColumnSpatial::filter(const IColumn::Filter& filter) { + column_match_filter_size(size(), filter.size()); + auto filtered = this->filter(filter, -1); + auto& spatial = assert_cast(*filtered); + clear(); + insert_range_from(spatial, 0, spatial.size()); + return size(); +} + +MutableColumnPtr ColumnSpatial::permute(const IColumn::Permutation& perm, size_t limit) const { + limit = limit ? std::min(size(), limit) : size(); + if (perm.size() < limit) { + throw Exception(ErrorCode::INTERNAL_ERROR, "Size of permutation is less than required"); + } + auto result = create(_primitive_type); + for (size_t i = 0; i < limit; ++i) { + const auto value = get_data_at(perm[i]); + result->insert_data(value.data, value.size); + } + return result; +} + +void ColumnSpatial::replace_column_data(const IColumn& rhs, size_t row, size_t self_row) { + DCHECK_LT(self_row, size()); + const auto& spatial = assert_cast(rhs); + DCHECK_EQ(_primitive_type, spatial._primitive_type); + const auto value = spatial.get_data_at(row); + if (value.size <= StringView::kInlineSize) { + _data[self_row] = StringView(value.data, cast_set(value.size)); + } else { + _data[self_row] = + StringView(_arena.insert(value.data, value.size), cast_set(value.size)); + } +} + +size_t ColumnSpatial::get_max_row_byte_size() const { + size_t maximum = 0; + for (const auto& value : _data) { + maximum = std::max(maximum, static_cast(value.size())); + } + return maximum + sizeof(uint32_t); +} + +size_t ColumnSpatial::deserialize_impl(const char* pos) { + const auto value_size = unaligned_load(pos); + pos += sizeof(value_size); + insert_data(pos, value_size); + return value_size + sizeof(value_size); +} + +size_t ColumnSpatial::serialize_impl(char* pos, size_t row) const { + const auto value = _data[row]; + const auto value_size = value.size(); + memcpy_fixed(pos, reinterpret_cast(&value_size)); + memcpy(pos + sizeof(uint32_t), value.data(), value_size); + return value_size + sizeof(uint32_t); +} + +size_t ColumnSpatial::serialize_size_at(size_t row) const { + return _data[row].size() + sizeof(uint32_t); +} + +StringRef ColumnSpatial::serialize_value_into_arena(size_t n, Arena& arena, + char const*& begin) const { + char* position = arena.alloc_continue(serialize_size_at(n), begin); + return {position, serialize_impl(position, n)}; +} + +const char* ColumnSpatial::deserialize_and_insert_from_arena(const char* pos) { + return pos + deserialize_impl(pos); +} + +void ColumnSpatial::serialize_vec(StringRef* keys, size_t num_rows) const { + for (size_t i = 0; i < num_rows; ++i) { + keys[i].size += serialize_impl(const_cast(keys[i].data + keys[i].size), i); + } +} + +void ColumnSpatial::deserialize_vec(StringRef* keys, size_t num_rows) { + for (size_t i = 0; i < num_rows; ++i) { + const auto size = deserialize_impl(keys[i].data); + keys[i].data += size; + keys[i].size -= size; + } +} + +template +struct ColumnSpatial::less { + const ColumnSpatial& parent; + bool operator()(size_t lhs, size_t rhs) const { + const int comparison = parent._data[lhs].compare(parent._data[rhs]); + return positive ? comparison < 0 : comparison > 0; + } +}; + +void ColumnSpatial::get_permutation(bool reverse, size_t /* limit */, int /* nan_direction_hint */, + HybridSorter& sorter, IColumn::Permutation& result) const { + result.resize(size()); + for (size_t i = 0; i < size(); ++i) { + result[i] = i; + } + if (reverse) { + sorter.sort(result.begin(), result.end(), less {*this}); + } else { + sorter.sort(result.begin(), result.end(), less {*this}); + } +} + +void ColumnSpatial::insert_many_strings(const StringRef* strings, size_t num) { + for (size_t i = 0; i < num; ++i) { + insert_data(strings[i].data, strings[i].size); + } +} + +void ColumnSpatial::insert_many_strings_overflow(const StringRef* strings, size_t num, + size_t /* max_length */) { + insert_many_strings(strings, num); +} + +void ColumnSpatial::sort_column(const ColumnSorter* sorter, EqualFlags& flags, + IColumn::Permutation& perms, EqualRange& range, + bool last_column) const { + sorter->sort_column(*this, flags, perms, range, last_column); +} + +} // namespace doris diff --git a/be/src/core/column/column_spatial.h b/be/src/core/column/column_spatial.h new file mode 100644 index 00000000000000..35085c025a8c9d --- /dev/null +++ b/be/src/core/column/column_spatial.h @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include + +#include "core/arena.h" +#include "core/assert_cast.h" +#include "core/column/column.h" +#include "core/data_type/define_primitive_type.h" +#include "core/string_view.h" + +namespace doris { + +// Stores the raw OGC WKB payload for GEOMETRY and GEOGRAPHY. The primitive type is +// part of the column identity so spatial values cannot be substituted for VARBINARY. +class ColumnSpatial final : public COWHelper { +private: + using Self = ColumnSpatial; + friend class COWHelper; + template + struct less; + +public: + using Container = PaddedPODArray; + +private: + explicit ColumnSpatial(PrimitiveType primitive_type) : _primitive_type(primitive_type) { + DCHECK(primitive_type == TYPE_GEOMETRY || primitive_type == TYPE_GEOGRAPHY); + } + ColumnSpatial(const ColumnSpatial& src) : _primitive_type(src._primitive_type) { + _data.reserve(src._data.size()); + for (const auto& value : src._data) { + insert_data(value.data(), value.size()); + } + } + +public: + std::string get_name() const override { return "ColumnSpatial"; } + PrimitiveType get_primitive_type() const { return _primitive_type; } + size_t size() const override { return _data.size(); } + const Container& get_data() const { return _data; } + void resize(size_t n) override { _data.resize(n); } + void clear() override { + _data.clear(); + _arena.clear(); + } + Field operator[](size_t n) const override { + return Field::create_field(_data[n]); + } + void get(size_t n, Field& res) const override { + res = Field::create_field(_data[n]); + } + StringRef get_data_at(size_t n) const override { return _data[n].to_string_ref(); } + void insert(const Field& x) override { + const auto& value = x.get(); + insert_data(value.data(), value.size()); + } + void insert_from(const IColumn& src, size_t n) override { + const auto& spatial = assert_cast(src); + DCHECK_EQ(_primitive_type, spatial._primitive_type); + const auto value = spatial.get_data_at(n); + insert_data(value.data, value.size); + } + void insert_data(const char* pos, size_t length) override; + void insert_default() override { _data.push_back(doris::StringView()); } + int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override; + void get_permutation(bool reverse, size_t limit, int nan_direction_hint, HybridSorter& sorter, + IColumn::Permutation& res) const override; + size_t get_max_row_byte_size() const override; + void deserialize_vec(StringRef* keys, size_t num_rows) override; + void serialize_vec(StringRef* keys, size_t num_rows) const override; + void pop_back(size_t n) override { resize(size() - n); } + StringRef serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const override; + const char* deserialize_and_insert_from_arena(const char* pos) override; + void insert_range_from(const IColumn& src, size_t start, size_t length) override; + MutableColumnPtr clone_resized(size_t size) const override; + void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; + size_t allocated_bytes() const override { return _data.allocated_bytes() + _arena.size(); } + size_t byte_size() const override { + return _data.size() * sizeof(doris::StringView) + _arena.used_size(); + } + bool has_enough_capacity(const IColumn& src) const override; + ColumnPtr filter(const IColumn::Filter& filt, ssize_t result_size_hint) const override; + size_t filter(const IColumn::Filter& filter) override; + MutableColumnPtr permute(const IColumn::Permutation& perm, size_t limit) const override; + void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override; + void insert_many_strings(const StringRef* strings, size_t num) override; + void insert_many_strings_overflow(const StringRef* strings, size_t num, + size_t max_length) override; + void sort_column(const ColumnSorter* sorter, EqualFlags& flags, IColumn::Permutation& perms, + EqualRange& range, bool last_column) const override; + +private: + size_t deserialize_impl(const char* pos) override; + size_t serialize_impl(char* pos, size_t row) const override; + size_t serialize_size_at(size_t row) const override; + Container _data; + Arena _arena; + const PrimitiveType _primitive_type; +}; + +} // namespace doris diff --git a/be/src/core/data_type/data_type_factory.cpp b/be/src/core/data_type/data_type_factory.cpp index 19aa8383524206..63e840944b2981 100644 --- a/be/src/core/data_type/data_type_factory.cpp +++ b/be/src/core/data_type/data_type_factory.cpp @@ -56,6 +56,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_quantilestate.h" +#include "core/data_type/data_type_spatial.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_time.h" @@ -495,6 +496,12 @@ DataTypePtr DataTypeFactory::create_data_type(const PrimitiveType primitive_type case TYPE_VARBINARY: nested = std::make_shared(len, TYPE_VARBINARY); break; + case TYPE_GEOMETRY: + nested = std::make_shared(TYPE_GEOMETRY); + break; + case TYPE_GEOGRAPHY: + nested = std::make_shared(TYPE_GEOGRAPHY, "OGC:CRS84", "spherical"); + break; case TYPE_AGG_STATE: case TYPE_ARRAY: case TYPE_MAP: @@ -525,6 +532,23 @@ DataTypePtr DataTypeFactory::create_data_type(const std::vector& type case TTypeNodeType::SCALAR: { DCHECK(node.__isset.scalar_type); const TScalarType& scalar_type = node.scalar_type; + if (scalar_type.type == TPrimitiveType::GEOMETRY || + scalar_type.type == TPrimitiveType::GEOGRAPHY) { + if (!scalar_type.__isset.spatial_crs) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Missing spatial CRS in type descriptor"); + } + if (scalar_type.type == TPrimitiveType::GEOGRAPHY && + !scalar_type.__isset.spatial_algorithm) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Missing geography edge algorithm in type descriptor"); + } + const auto primitive_type = thrift_to_type(scalar_type.type); + DataTypePtr spatial_type = std::make_shared( + primitive_type, scalar_type.spatial_crs, + scalar_type.__isset.spatial_algorithm ? scalar_type.spatial_algorithm : ""); + return is_nullable ? make_nullable(spatial_type) : spatial_type; + } if (scalar_type.type == TPrimitiveType::VARIANT) { DCHECK(scalar_type.variant_max_subcolumns_count >= 0) << "count is: " << scalar_type.variant_max_subcolumns_count; @@ -649,6 +673,12 @@ DataTypePtr DataTypeFactory::create_data_type( nested = std::make_shared(node.variant_max_subcolumns_count(), node.variant_enable_doc_mode()); } + } else if (primitive_type == TYPE_GEOMETRY || primitive_type == TYPE_GEOGRAPHY) { + const std::string crs = node.has_spatial_crs() ? node.spatial_crs() : "OGC:CRS84"; + const std::string algorithm = primitive_type == TYPE_GEOGRAPHY + ? (node.has_spatial_algorithm() ? node.spatial_algorithm() : "spherical") + : ""; + nested = std::make_shared(primitive_type, crs, algorithm); } else { return create_data_type(primitive_type, is_nullable, scalar_type.has_precision() ? scalar_type.precision() : 0, diff --git a/be/src/core/data_type/data_type_spatial.cpp b/be/src/core/data_type/data_type_spatial.cpp new file mode 100644 index 00000000000000..973e30981c0e29 --- /dev/null +++ b/be/src/core/data_type/data_type_spatial.cpp @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_spatial.h" + +#include + +#include "agent/be_exec_version_manager.h" +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/data_type/data_type.h" +#include "core/field.h" +#include "core/string_view.h" + +namespace doris { + +doris::FieldType DataTypeSpatial::get_storage_field_type() const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "Spatial types are supported only by external Iceberg tables"); + return FieldType::OLAP_FIELD_TYPE_UNKNOWN; +} + +MutableColumnPtr DataTypeSpatial::create_column() const { + return ColumnSpatial::create(_primitive_type); +} + +Status DataTypeSpatial::check_column(const IColumn& column) const { + const IColumn* nested_column = &column; + if (is_column_const(column)) { + nested_column = &assert_cast(column).get_data_column(); + } + const auto* spatial = check_and_get_column(nested_column); + if (spatial == nullptr || spatial->get_primitive_type() != _primitive_type) { + return Status::InvalidArgument("Expected {} spatial column, got {}", get_name(), + column.get_name()); + } + return Status::OK(); +} + +int64_t DataTypeSpatial::get_uncompressed_serialized_bytes(const IColumn& column, + int be_exec_version) const { + DCHECK(be_exec_version >= USE_CONST_SERDE); + const IColumn* data_column = &column; + const bool is_const = is_column_const(column); + const size_t stored_rows = is_const ? 1 : column.size(); + if (is_const) { + data_column = &assert_cast(column).get_data_column(); + } + const auto& spatial = assert_cast(*data_column); + size_t payload_size = 0; + for (size_t i = 0; i < stored_rows; ++i) { + payload_size += spatial.get_data_at(i).size; + } + return sizeof(bool) + sizeof(size_t) * (2 + stored_rows) + payload_size; +} + +char* DataTypeSpatial::serialize(const IColumn& column, char* buf, int be_exec_version) const { + DCHECK(be_exec_version >= USE_CONST_SERDE); + const IColumn* data_column = &column; + size_t stored_rows = 0; + buf = serialize_const_flag_and_row_num(&data_column, buf, &stored_rows); + const auto& spatial = assert_cast(*data_column); + auto* sizes = reinterpret_cast(buf); + for (size_t i = 0; i < stored_rows; ++i) { + unaligned_store(&sizes[i], spatial.get_data_at(i).size); + } + char* payload = buf + sizeof(size_t) * stored_rows; + for (size_t i = 0; i < stored_rows; ++i) { + const auto value = spatial.get_data_at(i); + memcpy(payload, value.data, value.size); + payload += value.size; + } + return payload; +} + +const char* DataTypeSpatial::deserialize(const char* buf, MutableColumnPtr* column, + int be_exec_version) const { + DCHECK(be_exec_version >= USE_CONST_SERDE); + auto* original = column->get(); + size_t stored_rows = 0; + buf = deserialize_const_flag_and_row_num(buf, column, &stored_rows); + auto& spatial = assert_cast(*original); + const auto* sizes = reinterpret_cast(buf); + const char* payload = buf + sizeof(size_t) * stored_rows; + for (size_t i = 0; i < stored_rows; ++i) { + const size_t size = unaligned_load(&sizes[i]); + spatial.insert_data(payload, size); + payload += size; + } + return payload; +} + +Field DataTypeSpatial::get_field(const TExprNode& /* node */) const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "Spatial literals must be constructed from WKB by a spatial function"); +} + +FieldWithDataType DataTypeSpatial::get_field_with_data_type(const IColumn& column, + size_t row_num) const { + const auto value = assert_cast(column).get_data_at(row_num); + return FieldWithDataType {.field = Field::create_field(StringView(value)), + .base_scalar_type_id = _primitive_type}; +} + +bool DataTypeSpatial::equals(const IDataType& rhs) const { + const auto* other = dynamic_cast(&rhs); + return other != nullptr && _primitive_type == other->_primitive_type && _crs == other->_crs && + _algorithm == other->_algorithm; +} + +void DataTypeSpatial::to_protobuf(PTypeDesc* /* ptype */, PTypeNode* node, + PScalarType* /* scalar_type */) const { + node->set_spatial_crs(_crs); + if (_primitive_type == TYPE_GEOGRAPHY) { + node->set_spatial_algorithm(_algorithm); + } +} + +#ifdef BE_TEST +void DataTypeSpatial::to_thrift(TTypeDesc& thrift_type, TTypeNode& node) const { + IDataType::to_thrift(thrift_type, node); + node.scalar_type.__set_spatial_crs(_crs); + if (_primitive_type == TYPE_GEOGRAPHY) { + node.scalar_type.__set_spatial_algorithm(_algorithm); + } +} +#endif + +} // namespace doris diff --git a/be/src/core/data_type/data_type_spatial.h b/be/src/core/data_type/data_type_spatial.h new file mode 100644 index 00000000000000..247369bf2ec745 --- /dev/null +++ b/be/src/core/data_type/data_type_spatial.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "core/column/column_spatial.h" +#include "core/data_type/data_type.h" +#include "core/data_type_serde/data_type_varbinary_serde.h" + +namespace doris { + +// The physical payload is WKB, but this DataType deliberately remains distinct +// from VARBINARY so that the external-format reader can enforce spatial semantics. +class DataTypeSpatial final : public IDataType { +public: + DataTypeSpatial(PrimitiveType primitive_type, std::string crs = "OGC:CRS84", + std::string algorithm = "") + : _primitive_type(primitive_type), + _crs(std::move(crs)), + _algorithm(std::move(algorithm)) { + DCHECK(primitive_type == TYPE_GEOMETRY || primitive_type == TYPE_GEOGRAPHY); + DCHECK(!_crs.empty()); + DCHECK(primitive_type == TYPE_GEOMETRY || !_algorithm.empty()); + DCHECK(primitive_type == TYPE_GEOGRAPHY || _algorithm.empty()); + } + + const std::string get_family_name() const override { + return _primitive_type == TYPE_GEOMETRY ? "Geometry" : "Geography"; + } + PrimitiveType get_primitive_type() const override { return _primitive_type; } + const std::string& crs() const { return _crs; } + const std::string& algorithm() const { return _algorithm; } + + doris::FieldType get_storage_field_type() const override; + int64_t get_uncompressed_serialized_bytes(const IColumn& column, + int be_exec_version) const override; + char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; + const char* deserialize(const char* buf, MutableColumnPtr* column, + int be_exec_version) const override; + MutableColumnPtr create_column() const override; + Status check_column(const IColumn& column) const override; + Field get_field(const TExprNode& node) const override; + FieldWithDataType get_field_with_data_type(const IColumn& column, + size_t row_num) const override; + bool equals(const IDataType& rhs) const override; + DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override { + return std::make_shared(nesting_level); + } + void to_protobuf(PTypeDesc* ptype, PTypeNode* node, PScalarType* scalar_type) const override; +#ifdef BE_TEST + void to_thrift(TTypeDesc& thrift_type, TTypeNode& node) const override; +#endif + +private: + const PrimitiveType _primitive_type; + const std::string _crs; + const std::string _algorithm; +}; + +} // namespace doris diff --git a/be/src/core/data_type/define_primitive_type.h b/be/src/core/data_type/define_primitive_type.h index 852b68a96b5358..cebc41a77eda09 100644 --- a/be/src/core/data_type/define_primitive_type.h +++ b/be/src/core/data_type/define_primitive_type.h @@ -73,7 +73,9 @@ enum PrimitiveType : PrimitiveNative { TYPE_UINT64, /* 39, used as offset */ TYPE_FIXED_LENGTH_OBJECT, /* 40, represent fixed-length object on BE */ TYPE_VARBINARY, /* 41, varbinary */ - TYPE_TIMESTAMPTZ /* 42, timestamptz */ + TYPE_TIMESTAMPTZ, /* 42, timestamptz */ + TYPE_GEOMETRY, /* 43, Iceberg geometry */ + TYPE_GEOGRAPHY /* 44, Iceberg geography */ }; } // namespace doris diff --git a/be/src/core/data_type/primitive_type.cpp b/be/src/core/data_type/primitive_type.cpp index 746fc655323a22..2d1dd796f81325 100644 --- a/be/src/core/data_type/primitive_type.cpp +++ b/be/src/core/data_type/primitive_type.cpp @@ -141,6 +141,10 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { return TYPE_VARBINARY; case TPrimitiveType::TIMESTAMPTZ: return TYPE_TIMESTAMPTZ; + case TPrimitiveType::GEOMETRY: + return TYPE_GEOMETRY; + case TPrimitiveType::GEOGRAPHY: + return TYPE_GEOGRAPHY; default: CHECK(false) << ", meet unknown type " << ttype; return INVALID_TYPE; @@ -256,6 +260,10 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { return TPrimitiveType::VARBINARY; case TYPE_TIMESTAMPTZ: return TPrimitiveType::TIMESTAMPTZ; + case TYPE_GEOMETRY: + return TPrimitiveType::GEOMETRY; + case TYPE_GEOGRAPHY: + return TPrimitiveType::GEOGRAPHY; default: return TPrimitiveType::INVALID_TYPE; } @@ -373,6 +381,10 @@ std::string type_to_string(PrimitiveType t) { case TYPE_TIMESTAMPTZ: return "TIMESTAMPTZ"; + case TYPE_GEOMETRY: + return "GEOMETRY"; + case TYPE_GEOGRAPHY: + return "GEOGRAPHY"; default: return ""; }; diff --git a/be/src/core/data_type/storage_field_type.cpp b/be/src/core/data_type/storage_field_type.cpp index 269015401dcab7..8802efaeecf0dc 100644 --- a/be/src/core/data_type/storage_field_type.cpp +++ b/be/src/core/data_type/storage_field_type.cpp @@ -105,6 +105,8 @@ FieldType primitive_type_to_storage_field_type(PrimitiveType type) { case static_cast(33): // TYPE_LAMBDA_FUNCTION (deprecated) case PrimitiveType::TYPE_FIXED_LENGTH_OBJECT: case PrimitiveType::TYPE_VARBINARY: + case PrimitiveType::TYPE_GEOMETRY: + case PrimitiveType::TYPE_GEOGRAPHY: break; } diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp index 16ff1b20e8ac20..dd47ad61da4aac 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp @@ -20,9 +20,9 @@ #include #include "common/config.h" -#include "core/column/column_varbinary.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/parquet_decode_source.h" +#include "core/string_view.h" namespace doris { namespace { @@ -30,8 +30,7 @@ namespace { class VarbinaryParquetConsumer final : public ParquetFixedValueConsumer, public ParquetBinaryValueConsumer { public: - explicit VarbinaryParquetConsumer(IColumn& column) - : _column(assert_cast(column)) {} + explicit VarbinaryParquetConsumer(IColumn& column) : _column(column) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { for (size_t row = 0; row < num_values; ++row) { @@ -59,7 +58,7 @@ class VarbinaryParquetConsumer final : public ParquetFixedValueConsumer, } private: - ColumnVarbinary& _column; + IColumn& _column; }; } // namespace @@ -114,9 +113,9 @@ Status DataTypeVarbinarySerDe::write_column_to_mysql_binary(const IColumn& colum int64_t row_idx, bool col_const, const FormatOptions& options) const { auto col_index = index_check_const(row_idx, col_const); - const auto& data = assert_cast(column).get_data()[col_index]; + const auto data = column.get_data_at(col_index); - if (0 != result.push_string(data.data(), data.size())) { + if (0 != result.push_string(data.data, data.size)) { return Status::InternalError("pack mysql buffer failed."); } @@ -128,15 +127,14 @@ Status DataTypeVarbinarySerDe::write_column_to_arrow(const IColumn& column, cons int64_t start, int64_t end, const cctz::time_zone& ctz) const { auto lambda_function = [&](auto& builder) -> Status { - const auto& varbinary_column_data = assert_cast(column).get_data(); for (size_t i = start; i < end; ++i) { if (null_map && (*null_map)[i]) { RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, builder)); continue; } - const auto& string_view = varbinary_column_data[i]; - RETURN_IF_ERROR(checkArrowStatus(builder.Append(string_view.data(), string_view.size()), - column, builder)); + const auto value = column.get_data_at(i); + RETURN_IF_ERROR(checkArrowStatus( + builder.Append(value.data, cast_set(value.size)), column, builder)); } return Status::OK(); }; @@ -145,17 +143,16 @@ Status DataTypeVarbinarySerDe::write_column_to_arrow(const IColumn& column, cons return lambda_function(builder); } else if (array_builder->type()->id() == arrow::Type::LARGE_BINARY) { auto& builder = assert_cast(*array_builder); - const auto& varbinary_column_data = assert_cast(column).get_data(); for (size_t i = start; i < end; ++i) { if (null_map && (*null_map)[i]) { RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, builder)); continue; } - const auto& string_view = varbinary_column_data[i]; - RETURN_IF_ERROR(checkArrowStatus( - builder.Append(reinterpret_cast(string_view.data()), - cast_set(string_view.size())), - column, builder)); + const auto value = column.get_data_at(i); + RETURN_IF_ERROR( + checkArrowStatus(builder.Append(reinterpret_cast(value.data), + cast_set(value.size)), + column, builder)); } return Status::OK(); } else if (array_builder->type()->id() == arrow::Type::STRING) { @@ -165,20 +162,18 @@ Status DataTypeVarbinarySerDe::write_column_to_arrow(const IColumn& column, cons auto& builder = assert_cast(*array_builder); const int byte_width = static_cast(*array_builder->type()).byte_width(); - const auto& varbinary_column_data = assert_cast(column).get_data(); for (size_t i = start; i < end; ++i) { if (null_map && (*null_map)[i]) { RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, builder)); continue; } - const auto& string_view = varbinary_column_data[i]; - if (string_view.size() != byte_width) { + const auto value = column.get_data_at(i); + if (value.size != byte_width) { return Status::InvalidArgument("Fixed size binary column expects {} bytes, got {}", - byte_width, string_view.size()); + byte_width, value.size); } RETURN_IF_ERROR(checkArrowStatus( - builder.Append(reinterpret_cast(string_view.data())), column, - builder)); + builder.Append(reinterpret_cast(value.data)), column, builder)); } return Status::OK(); } else { @@ -192,7 +187,6 @@ Status DataTypeVarbinarySerDe::read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const { - auto& varbinary_column = assert_cast(column); if (arrow_array->type_id() == arrow::Type::STRING || arrow_array->type_id() == arrow::Type::BINARY) { const auto* concrete_array = assert_cast(arrow_array); @@ -215,10 +209,10 @@ Status DataTypeVarbinarySerDe::read_column_from_arrow(IColumn& column, if (config::enable_arrow_input_validation) { check_arrow_value_range(*concrete_array, start_offset, length, buffer_size); } - varbinary_column.insert_data( - reinterpret_cast(buffer->data() + start_offset), length); + column.insert_data(reinterpret_cast(buffer->data() + start_offset), + length); } else { - varbinary_column.insert_default(); + column.insert_default(); } } } else if (arrow_array->type_id() == arrow::Type::FIXED_SIZE_BINARY) { @@ -231,10 +225,10 @@ Status DataTypeVarbinarySerDe::read_column_from_arrow(IColumn& column, const uint32_t width = concrete_array->byte_width(); for (auto offset_i = start; offset_i < end; ++offset_i) { if (!concrete_array->IsNull(offset_i)) { - varbinary_column.insert_data( + column.insert_data( reinterpret_cast(concrete_array->GetValue(offset_i)), width); } else { - varbinary_column.insert_default(); + column.insert_default(); } } } else if (arrow_array->type_id() == arrow::Type::LARGE_STRING || @@ -254,10 +248,10 @@ Status DataTypeVarbinarySerDe::read_column_from_arrow(IColumn& column, check_arrow_value_range(*concrete_array, value_offset, value_length, buffer_size); } - varbinary_column.insert_data( - reinterpret_cast(buffer->data() + value_offset), value_length); + column.insert_data(reinterpret_cast(buffer->data() + value_offset), + value_length); } else { - varbinary_column.insert_default(); + column.insert_default(); } } } else { @@ -273,11 +267,10 @@ Status DataTypeVarbinarySerDe::write_column_to_orc(const std::string& timezone, int64_t start, int64_t end, Arena& arena, const FormatOptions& options) const { auto* cur_batch = dynamic_cast(orc_col_batch); - const auto& varbinary_column_data = assert_cast(column).get_data(); - for (auto row_id = start; row_id < end; row_id++) { - cur_batch->data[row_id] = const_cast(varbinary_column_data[row_id].data()); - cur_batch->length[row_id] = varbinary_column_data[row_id].size(); + const auto value = column.get_data_at(row_id); + cur_batch->data[row_id] = const_cast(value.data); + cur_batch->length[row_id] = value.size; } cur_batch->numElements = end - start; @@ -290,25 +283,25 @@ Status DataTypeVarbinarySerDe::serialize_one_cell_to_json(const IColumn& column, auto result = check_column_const_set_readability(column, row_num); ColumnPtr ptr = result.first; row_num = result.second; - const auto& value = assert_cast(*ptr).get_data_at(row_num); + const auto value = ptr->get_data_at(row_num); bw.write(value.data, value.size); return Status::OK(); } Status DataTypeVarbinarySerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const { - assert_cast(column).insert_data(slice.data, slice.size); + column.insert_data(slice.data, slice.size); return Status::OK(); } void DataTypeVarbinarySerDe::to_string(const IColumn& column, size_t row_num, BufferWritable& bw, const FormatOptions& options) const { - const auto& value = assert_cast(column).get_data()[row_num]; + const auto value = column.get_data_at(row_num); if (_nesting_level >= 2) { // in complex type, need to dump as hex string by hand - const auto& hex_str = value.dump_hex(); + const auto hex_str = StringView(value.data, cast_set(value.size)).dump_hex(); bw.write(hex_str.data(), hex_str.size()); } else { // mysql protocol will be handle as hex binary data directly - bw.write(value.data(), value.size()); + bw.write(value.data, value.size); } } diff --git a/be/src/exprs/function/geo/functions_geo.cpp b/be/src/exprs/function/geo/functions_geo.cpp index 6a191f133e542a..0adc895033935b 100644 --- a/be/src/exprs/function/geo/functions_geo.cpp +++ b/be/src/exprs/function/geo/functions_geo.cpp @@ -21,8 +21,10 @@ #include #include +#include #include +#include "common/cast_set.h" #include "common/compiler_util.h" #include "core/assert_cast.h" #include "core/block/block.h" @@ -32,15 +34,140 @@ #include "core/column/column_nullable.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_spatial.h" #include "core/data_type/data_type_string.h" #include "core/data_type/define_primitive_type.h" #include "core/string_ref.h" #include "exprs/function/geo/geo_common.h" #include "exprs/function/geo/geo_types.h" +#include "exprs/function/geo/wkb_parse.h" #include "exprs/function/simple_function_factory.h" +#include "exprs/function/string_hex_util.h" namespace doris { +static bool is_spatial_type(const DataTypePtr& type) { + const auto primitive_type = remove_nullable(type)->get_primitive_type(); + return primitive_type == TYPE_GEOMETRY || primitive_type == TYPE_GEOGRAPHY; +} + +static bool is_geometry_type(const DataTypePtr& type) { + return remove_nullable(type)->get_primitive_type() == TYPE_GEOMETRY; +} + +static Status validate_geography_semantics(const DataTypePtr& type, const char* function_name) { + const auto& nested_type = remove_nullable(type); + if (!is_spatial_type(nested_type)) { + return Status::OK(); + } + + const auto* spatial_type = dynamic_cast(nested_type.get()); + DCHECK(spatial_type != nullptr); + if (spatial_type != nullptr && spatial_type->get_primitive_type() == TYPE_GEOGRAPHY && + spatial_type->crs() == "OGC:CRS84" && spatial_type->algorithm() == "spherical") { + return Status::OK(); + } + return Status::NotSupported( + "Function {} requires GEOGRAPHY(OGC:CRS84, spherical) for spatial inputs", + function_name); +} + +static std::unique_ptr decode_geo_shape(StringRef value, const DataTypePtr& type, + GeoParseStatus* parse_status = nullptr) { + if (!is_spatial_type(type)) { + return GeoShape::from_encoded(value.data, value.size); + } + + GeoParseStatus status; + auto shape = GeoShape::from_wkb_bytes(value.data, value.size, status); + if (parse_status != nullptr) { + *parse_status = status; + } + return status == GEO_PARSE_OK ? std::move(shape) : nullptr; +} + +static bool has_unsupported_spatial_wkb_metadata(StringRef value) { + if (value.size < 5) { + return false; + } + + const auto byte_order = static_cast(value.data[0]); + if (byte_order != 0 && byte_order != 1) { + return false; + } + + const auto byte_at = [&value](size_t offset) { + return static_cast(value.data[offset]); + }; + const uint32_t type = byte_order == 1 ? static_cast(byte_at(1)) | + (static_cast(byte_at(2)) << 8) | + (static_cast(byte_at(3)) << 16) | + (static_cast(byte_at(4)) << 24) + : (static_cast(byte_at(1)) << 24) | + (static_cast(byte_at(2)) << 16) | + (static_cast(byte_at(3)) << 8) | + static_cast(byte_at(4)); + + constexpr uint32_t ewkb_z_flag = 0x80000000; + constexpr uint32_t ewkb_m_flag = 0x40000000; + constexpr uint32_t ewkb_srid_flag = 0x20000000; + constexpr uint32_t ewkb_metadata_flags = ewkb_z_flag | ewkb_m_flag | ewkb_srid_flag; + if ((type & ewkb_metadata_flags) != 0) { + return true; + } + + return type >= 1000 && type < 4000; +} + +static bool decode_wkb_hex(StringRef value, std::string* wkb) { + const char* data = value.data; + size_t size = value.size; + if (size >= 2 && ((data[0] == '0' && data[1] == 'x') || (data[0] == '\\' && data[1] == 'x'))) { + data += 2; + size -= 2; + } + if (size == 0 || (size & 1) != 0) { + return false; + } + wkb->resize(size / 2); + return string_hex::hex_decode(data, cast_set(size), wkb->data()) == + size / 2; +} + +Status validate_spatial_wkb_inputs(const Block& block, const ColumnNumbers& arguments) { + for (const auto argument : arguments) { + const auto& column = block.get_by_position(argument).column; + const auto& type = block.get_data_type(argument); + if (!is_spatial_type(type)) { + continue; + } + for (size_t row = 0; row < column->size(); ++row) { + if (column->is_null_at(row)) { + continue; + } + const auto value = column->get_data_at(row); + if (has_unsupported_spatial_wkb_metadata(value)) { + return Status::NotSupported( + "WKB dimensions or embedded SRID are not supported for spatial inputs at " + "row {}", + row); + } + const auto parse_status = remove_nullable(type)->get_primitive_type() == TYPE_GEOMETRY + ? WkbParse::validate_wkb_bytes(value.data, value.size) + : [&] { + GeoParseStatus status; + decode_geo_shape(value, type, &status); + return status; + }(); + if (parse_status != GEO_PARSE_OK) { + return Status::InvalidArgument("Invalid WKB in spatial input at row {}: {}", row, + to_string(parse_status)); + } + } + } + return Status::OK(); +} + struct StPoint { static constexpr auto NAME = "st_point"; static const size_t NUM_ARGS = 2; @@ -94,6 +221,7 @@ struct StAsText { auto return_type = block.get_data_type(result); auto& input = block.get_by_position(arguments[0]).column; + const auto& input_type = block.get_data_type(arguments[0]); auto size = input->size(); @@ -104,7 +232,18 @@ struct StAsText { std::unique_ptr shape; for (int row = 0; row < size; ++row) { auto shape_value = input->get_data_at(row); - shape = GeoShape::from_encoded(shape_value.data, shape_value.size); + if (is_geometry_type(input_type)) { + std::string wkt; + if (WkbParse::wkb_to_wkt(shape_value.data, shape_value.size, &wkt) == + GEO_PARSE_OK) { + res->insert_data(wkt.data(), wkt.size()); + continue; + } + null_map_data[row] = 1; + res->insert_default(); + continue; + } + shape = decode_geo_shape(shape_value, input_type); if (shape == nullptr) { null_map_data[row] = 1; res->insert_default(); @@ -129,6 +268,7 @@ struct StX { auto return_type = block.get_data_type(result); auto& input = block.get_by_position(arguments[0]).column; + const auto& input_type = block.get_data_type(arguments[0]); auto size = input->size(); @@ -137,17 +277,28 @@ struct StX { auto& null_map_data = null_map->get_data(); res->reserve(size); - GeoPoint point; for (int row = 0; row < size; ++row) { auto point_value = input->get_data_at(row); - auto pt = point.decode_from(point_value.data, point_value.size); - - if (!pt) { + if (is_geometry_type(input_type)) { + double x; + double y; + if (WkbParse::point_coordinates(point_value.data, point_value.size, &x, &y) == + GEO_PARSE_OK) { + res->insert_value(x); + continue; + } + null_map_data[row] = 1; + res->insert_default(); + continue; + } + auto shape = decode_geo_shape(point_value, input_type); + auto* point = shape ? dynamic_cast(shape.get()) : nullptr; + if (point == nullptr) { null_map_data[row] = 1; res->insert_default(); continue; } - auto x_value = point.x(); + auto x_value = point->x(); res->insert_value(x_value); } block.replace_by_position(result, @@ -166,6 +317,7 @@ struct StY { auto return_type = block.get_data_type(result); auto& input = block.get_by_position(arguments[0]).column; + const auto& input_type = block.get_data_type(arguments[0]); auto size = input->size(); @@ -174,17 +326,28 @@ struct StY { auto null_map = ColumnUInt8::create(size, 0); auto& null_map_data = null_map->get_data(); - GeoPoint point; for (int row = 0; row < size; ++row) { auto point_value = input->get_data_at(row); - auto pt = point.decode_from(point_value.data, point_value.size); - - if (!pt) { + if (is_geometry_type(input_type)) { + double x; + double y; + if (WkbParse::point_coordinates(point_value.data, point_value.size, &x, &y) == + GEO_PARSE_OK) { + res->insert_value(y); + continue; + } + null_map_data[row] = 1; + res->insert_default(); + continue; + } + auto shape = decode_geo_shape(point_value, input_type); + auto* point = shape ? dynamic_cast(shape.get()) : nullptr; + if (point == nullptr) { null_map_data[row] = 1; res->insert_default(); continue; } - auto y_value = point.y(); + auto y_value = point->y(); res->insert_value(y_value); } block.replace_by_position(result, @@ -274,45 +437,47 @@ struct StAngle { DCHECK_EQ(arguments.size(), 3); auto return_type = block.get_data_type(result); - auto p1 = ColumnView::create(block.get_by_position(arguments[0]).column); - auto p2 = ColumnView::create(block.get_by_position(arguments[1]).column); - auto p3 = ColumnView::create(block.get_by_position(arguments[2]).column); - const auto size = p1.size(); + const auto& p1 = block.get_by_position(arguments[0]).column; + const auto& p2 = block.get_by_position(arguments[1]).column; + const auto& p3 = block.get_by_position(arguments[2]).column; + const auto& p1_type = block.get_data_type(arguments[0]); + const auto& p2_type = block.get_data_type(arguments[1]); + const auto& p3_type = block.get_data_type(arguments[2]); + RETURN_IF_ERROR(validate_geography_semantics(p1_type, NAME)); + RETURN_IF_ERROR(validate_geography_semantics(p2_type, NAME)); + RETURN_IF_ERROR(validate_geography_semantics(p3_type, NAME)); + const auto size = p1->size(); auto res = ColumnFloat64::create(); res->reserve(size); auto null_map = ColumnUInt8::create(size, 0); auto& null_map_data = null_map->get_data(); - GeoPoint point1; - GeoPoint point2; - GeoPoint point3; - for (int row = 0; row < size; ++row) { - auto shape_value1 = p1.value_at(row); - auto pt1 = point1.decode_from(shape_value1.data, shape_value1.size); - if (!pt1) { + auto point1 = decode_geo_shape(p1->get_data_at(row), p1_type); + auto* pt1 = point1 ? dynamic_cast(point1.get()) : nullptr; + if (pt1 == nullptr) { null_map_data[row] = 1; res->insert_default(); continue; } - auto shape_value2 = p2.value_at(row); - auto pt2 = point2.decode_from(shape_value2.data, shape_value2.size); - if (!pt2) { + auto point2 = decode_geo_shape(p2->get_data_at(row), p2_type); + auto* pt2 = point2 ? dynamic_cast(point2.get()) : nullptr; + if (pt2 == nullptr) { null_map_data[row] = 1; res->insert_default(); continue; } - auto shape_value3 = p3.value_at(row); - auto pt3 = point3.decode_from(shape_value3.data, shape_value3.size); - if (!pt3) { + auto point3 = decode_geo_shape(p3->get_data_at(row), p3_type); + auto* pt3 = point3 ? dynamic_cast(point3.get()) : nullptr; + if (pt3 == nullptr) { null_map_data[row] = 1; res->insert_default(); continue; } double angle = 0; - if (!GeoPoint::ComputeAngle(&point1, &point2, &point3, &angle)) { + if (!GeoPoint::ComputeAngle(pt1, pt2, pt3, &angle)) { null_map_data[row] = 1; res->insert_default(); continue; @@ -333,22 +498,23 @@ struct StAzimuth { DCHECK_EQ(arguments.size(), 2); auto return_type = block.get_data_type(result); - auto left_col = ColumnView::create(block.get_by_position(arguments[0]).column); - auto right_col = - ColumnView::create(block.get_by_position(arguments[1]).column); + const auto& left_col = block.get_by_position(arguments[0]).column; + const auto& right_col = block.get_by_position(arguments[1]).column; + const auto& left_type = block.get_data_type(arguments[0]); + const auto& right_type = block.get_data_type(arguments[1]); + RETURN_IF_ERROR(validate_geography_semantics(left_type, NAME)); + RETURN_IF_ERROR(validate_geography_semantics(right_type, NAME)); - const auto size = left_col.size(); + const auto size = left_col->size(); auto res = ColumnFloat64::create(); res->reserve(size); auto null_map = ColumnUInt8::create(size, 0); auto& null_map_data = null_map->get_data(); - GeoPoint point1; - GeoPoint point2; for (int row = 0; row < size; ++row) { - auto shape_value1 = left_col.value_at(row); - auto pt1 = point1.decode_from(shape_value1.data, shape_value1.size); - auto shape_value2 = right_col.value_at(row); - auto pt2 = point2.decode_from(shape_value2.data, shape_value2.size); + auto point1 = decode_geo_shape(left_col->get_data_at(row), left_type); + auto point2 = decode_geo_shape(right_col->get_data_at(row), right_type); + auto* pt1 = point1 ? dynamic_cast(point1.get()) : nullptr; + auto* pt2 = point2 ? dynamic_cast(point2.get()) : nullptr; if (!(pt1 && pt2)) { null_map_data[row] = 1; @@ -357,7 +523,7 @@ struct StAzimuth { } double angle = 0; - if (!GeoPoint::ComputeAzimuth(&point1, &point2, &angle)) { + if (!GeoPoint::ComputeAzimuth(pt1, pt2, &angle)) { null_map_data[row] = 1; res->insert_default(); continue; @@ -379,6 +545,8 @@ struct StAreaSquareMeters { auto return_type = block.get_data_type(result); auto col = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); + const auto& input_type = block.get_data_type(arguments[0]); + RETURN_IF_ERROR(validate_geography_semantics(input_type, NAME)); const auto size = col->size(); auto res = ColumnFloat64::create(); res->reserve(size); @@ -388,7 +556,7 @@ struct StAreaSquareMeters { for (int row = 0; row < size; ++row) { auto shape_value = col->get_data_at(row); - shape = GeoShape::from_encoded(shape_value.data, shape_value.size); + shape = decode_geo_shape(shape_value, input_type); if (!shape) { null_map_data[row] = 1; res->insert_default(); @@ -419,6 +587,8 @@ struct StAreaSquareKm { auto return_type = block.get_data_type(result); auto col = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); + const auto& input_type = block.get_data_type(arguments[0]); + RETURN_IF_ERROR(validate_geography_semantics(input_type, NAME)); const auto size = col->size(); auto res = ColumnFloat64::create(); res->reserve(size); @@ -429,7 +599,7 @@ struct StAreaSquareKm { for (int row = 0; row < size; ++row) { auto shape_value = col->get_data_at(row); - shape = GeoShape::from_encoded(shape_value.data, shape_value.size); + shape = decode_geo_shape(shape_value, input_type); if (!shape) { null_map_data[row] = 1; res->insert_default(); @@ -504,24 +674,22 @@ struct StRelationFunction { static Status execute(Block& block, const ColumnNumbers& arguments, size_t result) { DCHECK_EQ(arguments.size(), 2); auto return_type = block.get_data_type(result); - auto left_col = ColumnView::create(block.get_by_position(arguments[0]).column); - auto right_col = - ColumnView::create(block.get_by_position(arguments[1]).column); + const auto& left_col = block.get_by_position(arguments[0]).column; + const auto& right_col = block.get_by_position(arguments[1]).column; + const auto& left_type = block.get_data_type(arguments[0]); + const auto& right_type = block.get_data_type(arguments[1]); + RETURN_IF_ERROR(validate_geography_semantics(left_type, NAME)); + RETURN_IF_ERROR(validate_geography_semantics(right_type, NAME)); - const auto size = left_col.size(); + const auto size = left_col->size(); auto res = ColumnUInt8::create(size, 0); auto null_map = ColumnUInt8::create(size, 0); auto& null_map_data = null_map->get_data(); for (int row = 0; row < size; ++row) { - auto lhs_value = left_col.value_at(row); - auto rhs_value = right_col.value_at(row); - - std::unique_ptr shape1( - GeoShape::from_encoded(lhs_value.data, lhs_value.size)); - std::unique_ptr shape2( - GeoShape::from_encoded(rhs_value.data, rhs_value.size)); + auto shape1 = decode_geo_shape(left_col->get_data_at(row), left_type); + auto shape2 = decode_geo_shape(right_col->get_data_at(row), right_type); if (!shape1 || !shape2) { null_map_data[row] = 1; @@ -629,21 +797,34 @@ struct StGeoFromText { struct StGeometryFromWKB { static constexpr auto NAME = "st_geometryfromwkb"; static constexpr GeoShapeType shape_type = GEO_SHAPE_ANY; + static constexpr PrimitiveType OUTPUT_TYPE = TYPE_GEOMETRY; }; struct StGeomFromWKB { static constexpr auto NAME = "st_geomfromwkb"; static constexpr GeoShapeType shape_type = GEO_SHAPE_ANY; + static constexpr PrimitiveType OUTPUT_TYPE = TYPE_GEOMETRY; +}; + +struct StGeogFromWKB { + static constexpr auto NAME = "st_geogfromwkb"; + static constexpr GeoShapeType shape_type = GEO_SHAPE_ANY; + static constexpr PrimitiveType OUTPUT_TYPE = TYPE_GEOGRAPHY; +}; + +struct StGeometryFromWKBTyped { + static constexpr auto NAME = "st_geometryfromwkbtyped"; + static constexpr GeoShapeType shape_type = GEO_SHAPE_ANY; + static constexpr PrimitiveType OUTPUT_TYPE = TYPE_GEOMETRY; }; template -struct StGeoFromWkb { +struct LegacyStGeoFromWkb { static constexpr auto NAME = Impl::NAME; static const size_t NUM_ARGS = 1; using Type = DataTypeString; static Status execute(Block& block, const ColumnNumbers& arguments, size_t result) { DCHECK_EQ(arguments.size(), 1); - auto return_type = block.get_data_type(result); auto& geo = block.get_by_position(arguments[0]).column; const auto size = geo->size(); @@ -653,8 +834,8 @@ struct StGeoFromWkb { GeoParseStatus status; std::string buf; for (int row = 0; row < size; ++row) { - auto value = geo->get_data_at(row); - std::unique_ptr shape = GeoShape::from_wkb(value.data, value.size, status); + const auto value = geo->get_data_at(row); + auto shape = GeoShape::from_wkb(value.data, value.size, status); if (shape == nullptr || status != GEO_PARSE_OK) { null_map_data[row] = 1; res->insert_default(); @@ -670,6 +851,71 @@ struct StGeoFromWkb { } }; +template +struct StGeoFromWkb { + static constexpr auto NAME = Impl::NAME; + static constexpr PrimitiveType OUTPUT_TYPE = Impl::OUTPUT_TYPE; + static const size_t NUM_ARGS = 1; + static Status execute(Block& block, const ColumnNumbers& arguments, size_t result) { + DCHECK_EQ(arguments.size(), 1); + auto& geo = block.get_by_position(arguments[0]).column; + + const auto size = geo->size(); + auto res = ColumnSpatial::create(Impl::OUTPUT_TYPE); + auto null_map = ColumnUInt8::create(size, 0); + auto& null_map_data = null_map->get_data(); + GeoParseStatus status; + std::string wkb; + for (int row = 0; row < size; ++row) { + auto value = geo->get_data_at(row); + if (!decode_wkb_hex(value, &wkb)) { + null_map_data[row] = 1; + res->insert_default(); + continue; + } + if (has_unsupported_spatial_wkb_metadata({wkb.data(), wkb.size()})) { + null_map_data[row] = 1; + res->insert_default(); + continue; + } + std::unique_ptr shape = + GeoShape::from_wkb_bytes(wkb.data(), wkb.size(), status); + if (shape == nullptr || status != GEO_PARSE_OK) { + null_map_data[row] = 1; + res->insert_default(); + continue; + } + res->insert_data(wkb.data(), wkb.size()); + } + block.replace_by_position(result, + ColumnNullable::create(std::move(res), std::move(null_map))); + return Status::OK(); + } +}; + +template +class SpatialWkbConstructorFunction : public IFunction { +public: + static constexpr auto name = Impl::NAME; + static FunctionPtr create() { return std::make_shared>(); } + String get_name() const override { return name; } + size_t get_number_of_arguments() const override { return Impl::NUM_ARGS; } + bool is_variadic() const override { return false; } + + DataTypePtr get_return_type_impl(const DataTypes&) const override { + if constexpr (Impl::OUTPUT_TYPE == TYPE_GEOGRAPHY) { + return make_nullable( + std::make_shared(TYPE_GEOGRAPHY, "OGC:CRS84", "spherical")); + } + return make_nullable(std::make_shared(TYPE_GEOMETRY)); + } + + Status execute_impl(FunctionContext*, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t) const override { + return Impl::execute(block, arguments, result); + } +}; + struct StAsBinary { static constexpr auto NAME = "st_asbinary"; static const size_t NUM_ARGS = 1; @@ -680,6 +926,7 @@ struct StAsBinary { auto res = ColumnString::create(); auto col = block.get_by_position(arguments[0]).column; + const auto& input_type = block.get_data_type(arguments[0]); const auto size = col->size(); auto null_map = ColumnUInt8::create(size, 0); auto& null_map_data = null_map->get_data(); @@ -687,7 +934,11 @@ struct StAsBinary { for (int row = 0; row < size; ++row) { auto shape_value = col->get_data_at(row); - shape = GeoShape::from_encoded(shape_value.data, shape_value.size); + if (is_geometry_type(input_type)) { + res->insert_data(shape_value.data, shape_value.size); + continue; + } + shape = decode_geo_shape(shape_value, input_type); if (!shape) { null_map_data[row] = 1; res->insert_default(); @@ -718,6 +969,8 @@ struct StLength { auto return_type = block.get_data_type(result); auto col = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); + const auto& input_type = block.get_data_type(arguments[0]); + RETURN_IF_ERROR(validate_geography_semantics(input_type, NAME)); const auto size = col->size(); auto res = ColumnFloat64::create(); res->reserve(size); @@ -727,7 +980,7 @@ struct StLength { std::unique_ptr shape; for (int row = 0; row < size; ++row) { auto shape_value = col->get_data_at(row); - shape = GeoShape::from_encoded(shape_value.data, shape_value.size); + shape = decode_geo_shape(shape_value, input_type); if (!shape) { null_map_data[row] = 1; res->insert_default(); @@ -753,6 +1006,7 @@ struct StGeometryType { auto return_type = block.get_data_type(result); auto col = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); + const auto& input_type = block.get_data_type(arguments[0]); const auto size = col->size(); auto res = ColumnString::create(); auto null_map = ColumnUInt8::create(size, 0); @@ -761,7 +1015,18 @@ struct StGeometryType { std::unique_ptr shape; for (int row = 0; row < size; ++row) { auto shape_value = col->get_data_at(row); - shape = GeoShape::from_encoded(shape_value.data, shape_value.size); + if (is_geometry_type(input_type)) { + std::string geo_type; + if (WkbParse::geometry_type(shape_value.data, shape_value.size, &geo_type) == + GEO_PARSE_OK) { + res->insert_data(geo_type.data(), geo_type.size()); + continue; + } + null_map_data[row] = 1; + res->insert_default(); + continue; + } + shape = decode_geo_shape(shape_value, input_type); if (!shape) { null_map_data[row] = 1; res->insert_default(); @@ -790,6 +1055,11 @@ struct StDistance { unpack_if_const(block.get_by_position(arguments[0]).column); const auto& [right_column, right_const] = unpack_if_const(block.get_by_position(arguments[1]).column); + const auto& left_type = block.get_data_type(arguments[0]); + const auto& right_type = block.get_data_type(arguments[1]); + + RETURN_IF_ERROR(validate_geography_semantics(left_type, NAME)); + RETURN_IF_ERROR(validate_geography_semantics(right_type, NAME)); const auto size = std::max(left_column->size(), right_column->size()); @@ -798,139 +1068,74 @@ struct StDistance { auto null_map = ColumnUInt8::create(size, 0); auto& null_map_data = null_map->get_data(); - if (left_const) { - const_vector(left_column, right_column, res, null_map_data, size); - } else if (right_const) { - vector_const(left_column, right_column, res, null_map_data, size); + if (left_const || right_const) { + const auto& const_column = left_const ? left_column : right_column; + const auto& const_type = left_const ? left_type : right_type; + auto const_shape = decode_geo_shape(const_column->get_data_at(0), const_type); + if (!const_shape) { + for (int row = 0; row < size; ++row) { + null_map_data[row] = 1; + res->insert_default(); + } + } else { + const auto& vector_column = left_const ? right_column : left_column; + const auto& vector_type = left_const ? right_type : left_type; + for (int row = 0; row < size; ++row) { + auto vector_shape = + decode_geo_shape(vector_column->get_data_at(row), vector_type); + if (!vector_shape) { + null_map_data[row] = 1; + res->insert_default(); + continue; + } + const double distance = left_const ? const_shape->Distance(vector_shape.get()) + : vector_shape->Distance(const_shape.get()); + if (UNLIKELY(distance < 0)) { + null_map_data[row] = 1; + res->insert_default(); + continue; + } + res->insert_value(distance); + } + } } else { - vector_vector(left_column, right_column, res, null_map_data, size); + for (int row = 0; row < size; ++row) { + auto left_shape = decode_geo_shape(left_column->get_data_at(row), left_type); + auto right_shape = decode_geo_shape(right_column->get_data_at(row), right_type); + if (!left_shape || !right_shape) { + null_map_data[row] = 1; + res->insert_default(); + continue; + } + const double distance = left_shape->Distance(right_shape.get()); + if (UNLIKELY(distance < 0)) { + null_map_data[row] = 1; + res->insert_default(); + continue; + } + res->insert_value(distance); + } } block.replace_by_position(result, ColumnNullable::create(std::move(res), std::move(null_map))); return Status::OK(); } - -private: - static bool decode_shape(const StringRef& value, std::unique_ptr& shape) { - shape = GeoShape::from_encoded(value.data, value.size); - return static_cast(shape); - } - - static void loop_do(StringRef& lhs_value, StringRef& rhs_value, - std::vector>& shapes, - ColumnFloat64::MutablePtr& res, NullMap& null_map, int row) { - StringRef* strs[2] = {&lhs_value, &rhs_value}; - for (int i = 0; i < 2; ++i) { - if (!decode_shape(*strs[i], shapes[i])) { - null_map[row] = 1; - res->insert_default(); - return; - } - } - double distance = shapes[0]->Distance(shapes[1].get()); - if (UNLIKELY(distance < 0)) { - null_map[row] = 1; - res->insert_default(); - return; - } - res->insert_value(distance); - } - - static void const_vector(const ColumnPtr& left_column, const ColumnPtr& right_column, - ColumnFloat64::MutablePtr& res, NullMap& null_map, const size_t size) { - const auto* left_string = assert_cast(left_column.get()); - const auto* right_string = assert_cast(right_column.get()); - - auto lhs_value = left_string->get_data_at(0); - std::unique_ptr lhs_shape; - if (!decode_shape(lhs_value, lhs_shape)) { - for (int row = 0; row < size; ++row) { - null_map[row] = 1; - res->insert_default(); - } - return; - } - - std::unique_ptr rhs_shape; - for (int row = 0; row < size; ++row) { - auto rhs_value = right_string->get_data_at(row); - if (!decode_shape(rhs_value, rhs_shape)) { - null_map[row] = 1; - res->insert_default(); - continue; - } - double distance = lhs_shape->Distance(rhs_shape.get()); - if (UNLIKELY(distance < 0)) { - null_map[row] = 1; - res->insert_default(); - continue; - } - res->insert_value(distance); - } - } - - static void vector_const(const ColumnPtr& left_column, const ColumnPtr& right_column, - ColumnFloat64::MutablePtr& res, NullMap& null_map, const size_t size) { - const auto* left_string = assert_cast(left_column.get()); - const auto* right_string = assert_cast(right_column.get()); - - auto rhs_value = right_string->get_data_at(0); - std::unique_ptr rhs_shape; - if (!decode_shape(rhs_value, rhs_shape)) { - for (int row = 0; row < size; ++row) { - null_map[row] = 1; - res->insert_default(); - } - return; - } - - std::unique_ptr lhs_shape; - for (int row = 0; row < size; ++row) { - auto lhs_value = left_string->get_data_at(row); - if (!decode_shape(lhs_value, lhs_shape)) { - null_map[row] = 1; - res->insert_default(); - continue; - } - double distance = lhs_shape->Distance(rhs_shape.get()); - if (UNLIKELY(distance < 0)) { - null_map[row] = 1; - res->insert_default(); - continue; - } - res->insert_value(distance); - } - } - - static void vector_vector(const ColumnPtr& left_column, const ColumnPtr& right_column, - ColumnFloat64::MutablePtr& res, NullMap& null_map, - const size_t size) { - const auto* left_string = assert_cast(left_column.get()); - const auto* right_string = assert_cast(right_column.get()); - - std::vector> shapes(2); - for (int row = 0; row < size; ++row) { - auto lhs_value = left_string->get_data_at(row); - auto rhs_value = right_string->get_data_at(row); - loop_do(lhs_value, rhs_value, shapes, res, null_map, row); - } - } }; void register_function_geo(SimpleFunctionFactory& factory) { factory.register_function>(); - factory.register_function>>(); - factory.register_function>>(); - factory.register_function>(); - factory.register_function>(); - factory.register_function>(); - factory.register_function>(); - factory.register_function>(); - factory.register_function>(); - factory.register_function>>(); - factory.register_function>>(); - factory.register_function>>(); - factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); factory.register_function>(); factory.register_function>>(); factory.register_function>>(); @@ -941,12 +1146,15 @@ void register_function_geo(SimpleFunctionFactory& factory) { factory.register_function>>(); factory.register_function>(); factory.register_function>(); - factory.register_function>>(); - factory.register_function>>(); - factory.register_function>(); - factory.register_function>(); - factory.register_function>(); - factory.register_function>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function< + SpatialWkbConstructorFunction>>(); + factory.register_function>>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); } } // namespace doris diff --git a/be/src/exprs/function/geo/functions_geo.h b/be/src/exprs/function/geo/functions_geo.h index c75525641e6b60..c03e05a9853b17 100644 --- a/be/src/exprs/function/geo/functions_geo.h +++ b/be/src/exprs/function/geo/functions_geo.h @@ -53,6 +53,8 @@ struct StContainsState { std::vector> shapes; }; +Status validate_spatial_wkb_inputs(const Block& block, const ColumnNumbers& arguments); + template class GeoFunction : public IFunction { public: @@ -62,7 +64,6 @@ class GeoFunction : public IFunction { String get_name() const override { return name; } size_t get_number_of_arguments() const override { return Impl::NUM_ARGS; } bool is_variadic() const override { return false; } - DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { return make_nullable(std::make_shared()); } @@ -73,4 +74,21 @@ class GeoFunction : public IFunction { } }; +// Spatial WKB columns must remain visible to the implementation so it can validate the +// serialized value before any file data is consumed. Do not use this wrapper for legacy +// geo functions or numeric constructors: their nullable arguments rely on the default +// null propagation supplied by IFunction. +template +class SpatialWkbGeoFunction : public GeoFunction { +public: + static FunctionPtr create() { return std::make_shared>(); } + bool use_default_implementation_for_nulls() const override { return false; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + RETURN_IF_ERROR(validate_spatial_wkb_inputs(block, arguments)); + return GeoFunction::execute_impl(context, block, arguments, result, input_rows_count); + } +}; + } // namespace doris diff --git a/be/src/exprs/function/geo/geo_common.cpp b/be/src/exprs/function/geo/geo_common.cpp index 3b6f9211a16c5e..12cb0f36f19ac6 100644 --- a/be/src/exprs/function/geo/geo_common.cpp +++ b/be/src/exprs/function/geo/geo_common.cpp @@ -41,6 +41,8 @@ std::string to_string(GeoParseStatus status) { return "Circle invalid"; case GEO_PARSE_WKT_SYNTAX_ERROR: return "WKT syntax error"; + case GEO_PARSE_WKB_SYNTAX_ERROR: + return "WKB syntax error"; default: return "Unknown"; } diff --git a/be/src/exprs/function/geo/geo_types.cpp b/be/src/exprs/function/geo/geo_types.cpp index 03dd5c080c398a..1167505faf1d81 100644 --- a/be/src/exprs/function/geo/geo_types.cpp +++ b/be/src/exprs/function/geo/geo_types.cpp @@ -428,6 +428,13 @@ std::unique_ptr GeoShape::from_wkb(const char* data, size_t size, return shape; } +std::unique_ptr GeoShape::from_wkb_bytes(const char* data, size_t size, + GeoParseStatus& status) { + std::unique_ptr shape; + status = WkbParse::parse_wkb_bytes(data, size, shape); + return shape; +} + std::unique_ptr GeoShape::from_encoded(const void* ptr, size_t size) { if (size < 2 || ((const char*)ptr)[0] != 0X00) { return nullptr; diff --git a/be/src/exprs/function/geo/geo_types.h b/be/src/exprs/function/geo/geo_types.h index f6d8e04e0182fa..0a78c9129da38e 100644 --- a/be/src/exprs/function/geo/geo_types.h +++ b/be/src/exprs/function/geo/geo_types.h @@ -56,6 +56,8 @@ class GeoShape { static std::unique_ptr from_wkb(const char* data, size_t size, GeoParseStatus& status); + static std::unique_ptr from_wkb_bytes(const char* data, size_t size, + GeoParseStatus& status); void encode_to(std::string* buf); bool decode_from(const void* data, size_t size); diff --git a/be/src/exprs/function/geo/wkb_parse.cpp b/be/src/exprs/function/geo/wkb_parse.cpp index 390807379d92e4..ad0a37d5b85965 100644 --- a/be/src/exprs/function/geo/wkb_parse.cpp +++ b/be/src/exprs/function/geo/wkb_parse.cpp @@ -78,6 +78,9 @@ unsigned char ASCIIHexToUChar(char val) { GeoParseStatus WkbParse::parse_wkb(std::istream& is, std::unique_ptr& shape) { WkbParseContext ctx; + // Legacy ST_GeometryFromWKB accepts EWKB with an embedded SRID. The SRID is + // intentionally ignored because the legacy encoded representation has no CRS field. + ctx.allow_ewkb_srid = true; WkbParse::read_hex(is, ctx); if (ctx.parse_status == GEO_PARSE_OK) { @@ -86,6 +89,201 @@ GeoParseStatus WkbParse::parse_wkb(std::istream& is, std::unique_ptr& return ctx.parse_status; } +GeoParseStatus WkbParse::parse_wkb_bytes(const char* data, size_t size, + std::unique_ptr& shape) { + WkbParseContext ctx; + std::istringstream wkb(std::string(data, size), std::ios_base::binary | std::ios_base::in); + WkbParse::read(wkb, ctx); + if (ctx.parse_status == GEO_PARSE_OK) { + shape = std::move(ctx.shape); + } + return ctx.parse_status; +} + +GeoParseStatus WkbParse::validate_wkb_bytes(const char* data, size_t size) { + if (size == 0) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + + try { + const auto byte_order = static_cast(data[0]); + WkbParseContext ctx; + if (byte_order == byteOrder::wkbNDR) { + ctx.dis = ByteOrderDataInStream(reinterpret_cast(data), size); + ctx.dis.setOrder(ByteOrderValues::ENDIAN_LITTLE); + } else if (byte_order == byteOrder::wkbXDR) { + ctx.dis = ByteOrderDataInStream(reinterpret_cast(data), size); + ctx.dis.setOrder(ByteOrderValues::ENDIAN_BIG); + } else { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + return validate_geometry(ctx) && ctx.dis.size() == 0 ? GEO_PARSE_OK + : GEO_PARSE_WKB_SYNTAX_ERROR; + } catch (...) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } +} + +GeoParseStatus WkbParse::initialize_context(const char* data, size_t size, WkbParseContext* ctx) { + if (size == 0) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + const auto byte_order = static_cast(data[0]); + if (byte_order == byteOrder::wkbNDR) { + ctx->dis = ByteOrderDataInStream(reinterpret_cast(data), size); + ctx->dis.setOrder(ByteOrderValues::ENDIAN_LITTLE); + } else if (byte_order == byteOrder::wkbXDR) { + ctx->dis = ByteOrderDataInStream(reinterpret_cast(data), size); + ctx->dis.setOrder(ByteOrderValues::ENDIAN_BIG); + } else { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + return GEO_PARSE_OK; +} + +GeoParseStatus WkbParse::read_geometry_type(WkbParseContext& ctx, uint32_t* type) { + if (ctx.dis.size() < 5) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + ctx.dis.readByte(); + const uint32_t type_int = ctx.dis.readUnsigned(); + constexpr uint32_t ewkb_z_flag = 0x80000000; + constexpr uint32_t ewkb_m_flag = 0x40000000; + constexpr uint32_t ewkb_srid_flag = 0x20000000; + if ((type_int & (ewkb_z_flag | ewkb_m_flag)) != 0 || (type_int >= 1000 && type_int < 4000)) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + if ((type_int & ewkb_srid_flag) != 0) { + if (!ctx.allow_ewkb_srid || ctx.dis.size() < sizeof(uint32_t)) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + ctx.srid = static_cast(ctx.dis.readUnsigned()); + } + *type = type_int & WKB_TYPE_MASK; + return GEO_PARSE_OK; +} + +bool WkbParse::read_wkt_coordinates(uint32_t size, WkbParseContext& ctx, std::ostream& os) { + if (size == 0 || size > ctx.dis.size() / (2 * sizeof(double))) { + return false; + } + for (uint32_t i = 0; i < size; ++i) { + if (i != 0) { + os << ", "; + } + os << ctx.dis.readDouble() << " " << ctx.dis.readDouble(); + } + return true; +} + +bool WkbParse::read_wkt_geometry(WkbParseContext& ctx, std::ostream& os) { + uint32_t type; + if (read_geometry_type(ctx, &type) != GEO_PARSE_OK) { + return false; + } + switch (type) { + case wkbType::wkbPoint: + os << "POINT ("; + if (!read_wkt_coordinates(1, ctx, os)) { + return false; + } + os << ")"; + return true; + case wkbType::wkbLine: { + const uint32_t size = ctx.dis.readUnsigned(); + os << "LINESTRING ("; + if (!read_wkt_coordinates(size, ctx, os)) { + return false; + } + os << ")"; + return true; + } + case wkbType::wkbPolygon: { + const uint32_t loops = ctx.dis.readUnsigned(); + if (loops == 0 || loops > ctx.dis.size() / sizeof(uint32_t)) { + return false; + } + os << "POLYGON ("; + for (uint32_t loop = 0; loop < loops; ++loop) { + if (loop != 0) { + os << ", "; + } + const uint32_t size = ctx.dis.readUnsigned(); + os << "("; + if (size < 3 || !read_wkt_coordinates(size, ctx, os)) { + return false; + } + os << ")"; + } + os << ")"; + return true; + } + default: + return false; + } +} + +GeoParseStatus WkbParse::wkb_to_wkt(const char* data, size_t size, std::string* wkt) { + WkbParseContext ctx; + if (initialize_context(data, size, &ctx) != GEO_PARSE_OK) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + try { + std::ostringstream os; + if (!read_wkt_geometry(ctx, os) || ctx.dis.size() != 0) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + *wkt = os.str(); + return GEO_PARSE_OK; + } catch (...) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } +} + +GeoParseStatus WkbParse::point_coordinates(const char* data, size_t size, double* x, double* y) { + WkbParseContext ctx; + if (initialize_context(data, size, &ctx) != GEO_PARSE_OK) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + try { + uint32_t type; + if (read_geometry_type(ctx, &type) != GEO_PARSE_OK || type != wkbType::wkbPoint || + ctx.dis.size() != 2 * sizeof(double)) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + *x = ctx.dis.readDouble(); + *y = ctx.dis.readDouble(); + return GEO_PARSE_OK; + } catch (...) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } +} + +GeoParseStatus WkbParse::geometry_type(const char* data, size_t size, std::string* type) { + WkbParseContext ctx; + if (initialize_context(data, size, &ctx) != GEO_PARSE_OK) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + uint32_t wkb_type; + if (read_geometry_type(ctx, &wkb_type) != GEO_PARSE_OK) { + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + switch (wkb_type) { + case wkbType::wkbPoint: + *type = "ST_POINT"; + break; + case wkbType::wkbLine: + *type = "ST_LINESTRING"; + break; + case wkbType::wkbPolygon: + *type = "ST_POLYGON"; + break; + default: + return GEO_PARSE_WKB_SYNTAX_ERROR; + } + return GEO_PARSE_OK; +} + void WkbParse::read_hex(std::istream& is, WkbParseContext& ctx) { // setup input/output stream std::stringstream os(std::ios_base::binary | std::ios_base::in | std::ios_base::out); @@ -155,7 +353,7 @@ void WkbParse::read(std::istream& is, WkbParseContext& ctx) { } std::unique_ptr shape = readGeometry(ctx); - if (!shape) { + if (!shape || ctx.dis.size() != 0) { ctx.parse_status = GEO_PARSE_WKB_SYNTAX_ERROR; return; } @@ -170,22 +368,11 @@ std::unique_ptr WkbParse::readGeometry(WkbParseContext& ctx) { return nullptr; } - // Skip the byte order as we've already handled it - ctx.dis.readByte(); - - uint32_t typeInt = ctx.dis.readUnsigned(); - - // Check if geometry has SRID - bool has_srid = (typeInt & WKB_SRID_FLAG) != 0; - - // Read SRID if present - if (has_srid) { - ctx.dis.readUnsigned(); // Read and store SRID if needed + uint32_t geometryType; + if (read_geometry_type(ctx, &geometryType) != GEO_PARSE_OK) { + return nullptr; } - // Get the base geometry type - uint32_t geometryType = typeInt & WKB_TYPE_MASK; - std::unique_ptr shape; switch (geometryType) { @@ -316,4 +503,52 @@ bool WkbParse::readCoordinate(WkbParseContext& ctx) { return true; } +bool WkbParse::validate_geometry(WkbParseContext& ctx) { + if (ctx.dis.size() < 5) { + return false; + } + ctx.dis.readByte(); + const uint32_t type = ctx.dis.readUnsigned(); + constexpr uint32_t ewkb_metadata_flags = 0xE0000000; + if ((type & ewkb_metadata_flags) != 0 || (type >= 1000 && type < 4000)) { + return false; + } + + switch (type & WKB_TYPE_MASK) { + case wkbType::wkbPoint: + return validate_coordinates(1, ctx); + case wkbType::wkbLine: { + const uint32_t size = ctx.dis.readUnsigned(); + return size > 0 && validate_coordinates(size, ctx); + } + case wkbType::wkbPolygon: { + const uint32_t loops = ctx.dis.readUnsigned(); + if (loops == 0 || loops > ctx.dis.size() / sizeof(uint32_t)) { + return false; + } + for (uint32_t loop = 0; loop < loops; ++loop) { + const uint32_t size = ctx.dis.readUnsigned(); + if (size < 3 || !validate_coordinates(size, ctx)) { + return false; + } + } + return true; + } + default: + return false; + } +} + +bool WkbParse::validate_coordinates(uint32_t size, WkbParseContext& ctx) { + constexpr size_t coordinate_size = 2 * sizeof(double); + if (size > ctx.dis.size() / coordinate_size) { + return false; + } + for (uint32_t coordinate = 0; coordinate < size; ++coordinate) { + ctx.dis.readDouble(); + ctx.dis.readDouble(); + } + return true; +} + } // namespace doris diff --git a/be/src/exprs/function/geo/wkb_parse.h b/be/src/exprs/function/geo/wkb_parse.h index c992e5a6fa32eb..cee605c44a9156 100644 --- a/be/src/exprs/function/geo/wkb_parse.h +++ b/be/src/exprs/function/geo/wkb_parse.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "exprs/function/geo/geo_common.h" #include "exprs/function/geo/wkt_parse_type.h" @@ -33,13 +34,6 @@ class GeoLine; class GeoPoint; class GeoPolygon; -// WKB format constants -// According to OpenGIS Implementation Specification: -// The high bit of the type value is set to 1 if the WKB contains a SRID. -// Reference: OpenGIS Implementation Specification for Geographic information - Simple feature access - Part 1: Common architecture -// Bit mask to check if WKB contains SRID -constexpr uint32_t WKB_SRID_FLAG = 0x20000000; - // The geometry type is stored in the least significant byte of the type value // Bit mask to extract the base geometry type constexpr uint32_t WKB_TYPE_MASK = 0xFF; @@ -47,6 +41,12 @@ constexpr uint32_t WKB_TYPE_MASK = 0xFF; class WkbParse { public: static GeoParseStatus parse_wkb(std::istream& is, std::unique_ptr& shape); + static GeoParseStatus parse_wkb_bytes(const char* data, size_t size, + std::unique_ptr& shape); + static GeoParseStatus validate_wkb_bytes(const char* data, size_t size); + static GeoParseStatus wkb_to_wkt(const char* data, size_t size, std::string* wkt); + static GeoParseStatus point_coordinates(const char* data, size_t size, double* x, double* y); + static GeoParseStatus geometry_type(const char* data, size_t size, std::string* type); private: static void read_hex(std::istream& is, WkbParseContext& ctx); @@ -66,6 +66,13 @@ class WkbParse { static GeoParseStatus minMemSize(int wkbType, uint64_t size, WkbParseContext& ctx); static bool readCoordinate(WkbParseContext& ctx); + + static bool validate_geometry(WkbParseContext& ctx); + static bool validate_coordinates(uint32_t size, WkbParseContext& ctx); + static GeoParseStatus initialize_context(const char* data, size_t size, WkbParseContext* ctx); + static bool read_wkt_geometry(WkbParseContext& ctx, std::ostream& os); + static bool read_wkt_coordinates(uint32_t size, WkbParseContext& ctx, std::ostream& os); + static GeoParseStatus read_geometry_type(WkbParseContext& ctx, uint32_t* type); }; } // namespace doris diff --git a/be/src/exprs/function/geo/wkb_parse_ctx.h b/be/src/exprs/function/geo/wkb_parse_ctx.h index 44a140675d2182..eb641383b2d94c 100644 --- a/be/src/exprs/function/geo/wkb_parse_ctx.h +++ b/be/src/exprs/function/geo/wkb_parse_ctx.h @@ -33,6 +33,8 @@ struct WkbParseContext { int srid; + bool allow_ewkb_srid = false; + std::unique_ptr shape = nullptr; doris::GeoParseStatus parse_status = doris::GEO_PARSE_OK; }; diff --git a/be/src/format/parquet/parquet_column_convert.cpp b/be/src/format/parquet/parquet_column_convert.cpp index 2472e2f70a9188..010fab65adfc0d 100644 --- a/be/src/format/parquet/parquet_column_convert.cpp +++ b/be/src/format/parquet/parquet_column_convert.cpp @@ -48,6 +48,8 @@ bool PhysicalToLogicalConverter::is_parquet_native_type(PrimitiveType type) { case TYPE_STRING: case TYPE_CHAR: case TYPE_VARCHAR: + case TYPE_GEOMETRY: + case TYPE_GEOGRAPHY: return true; default: return false; diff --git a/be/src/format/parquet/schema_desc.cpp b/be/src/format/parquet/schema_desc.cpp index d92be3013ce21e..f309ffb68f8780 100644 --- a/be/src/format/parquet/schema_desc.cpp +++ b/be/src/format/parquet/schema_desc.cpp @@ -28,6 +28,7 @@ #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_spatial.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" #include "format/table/table_format_reader.h" @@ -70,6 +71,23 @@ static int num_children_node(const tparquet::SchemaElement& schema) { return schema.__isset.num_children ? schema.num_children : 0; } +static std::string geography_algorithm_to_string( + tparquet::EdgeInterpolationAlgorithm::type algorithm) { + switch (algorithm) { + case tparquet::EdgeInterpolationAlgorithm::SPHERICAL: + return "spherical"; + case tparquet::EdgeInterpolationAlgorithm::VINCENTY: + return "vincenty"; + case tparquet::EdgeInterpolationAlgorithm::THOMAS: + return "thomas"; + case tparquet::EdgeInterpolationAlgorithm::ANDOYER: + return "andoyer"; + case tparquet::EdgeInterpolationAlgorithm::KARNEY: + return "karney"; + } + throw Exception(Status::InternalError("Unsupported Parquet geography edge algorithm")); +} + /** * `repeated_parent_def_level` is the definition level of the first ancestor node whose repetition_type equals REPEATED. * Empty array/map values are not stored in doris columns, so have to use `repeated_parent_def_level` to skip the @@ -329,6 +347,16 @@ std::pair FieldDescriptor::convert_to_doris_type( } } else if (logicalType.__isset.FLOAT16) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + } else if (logicalType.__isset.GEOMETRY) { + const auto& geometry = logicalType.GEOMETRY; + ans.first = std::make_shared( + TYPE_GEOMETRY, geometry.__isset.crs ? geometry.crs : "OGC:CRS84"); + } else if (logicalType.__isset.GEOGRAPHY) { + const auto& geography = logicalType.GEOGRAPHY; + ans.first = std::make_shared( + TYPE_GEOGRAPHY, geography.__isset.crs ? geography.crs : "OGC:CRS84", + geography.__isset.algorithm ? geography_algorithm_to_string(geography.algorithm) + : "spherical"); } else { throw Exception(Status::InternalError("Not supported parquet logicalType")); } diff --git a/be/src/format/table/iceberg/arrow_schema_util.cpp b/be/src/format/table/iceberg/arrow_schema_util.cpp index 84dc29f594de82..cd17aca5beffce 100644 --- a/be/src/format/table/iceberg/arrow_schema_util.cpp +++ b/be/src/format/table/iceberg/arrow_schema_util.cpp @@ -28,6 +28,9 @@ const char* ArrowSchemaUtil::PARQUET_FIELD_ID = "PARQUET:field_id"; const char* ArrowSchemaUtil::ORIGINAL_TYPE = "originalType"; const char* ArrowSchemaUtil::MAP_TYPE_VALUE = "mapType"; const char* ArrowSchemaUtil::UUID_TYPE_VALUE = "uuid"; +const char* ArrowSchemaUtil::ICEBERG_BINARY_TYPE = "iceberg.binary-type"; +const char* ArrowSchemaUtil::GEOMETRY_BINARY_TYPE_VALUE = "GEOMETRY"; +const char* ArrowSchemaUtil::GEOGRAPHY_BINARY_TYPE_VALUE = "GEOGRAPHY"; Status ArrowSchemaUtil::convert(const Schema* schema, const std::string& timezone, std::vector>& fields) { @@ -86,6 +89,16 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field, arrow_type = arrow::binary(); break; + case iceberg::TypeID::GEOMETRY: + metadata[ICEBERG_BINARY_TYPE] = GEOMETRY_BINARY_TYPE_VALUE; + arrow_type = arrow::binary(); + break; + + case iceberg::TypeID::GEOGRAPHY: + metadata[ICEBERG_BINARY_TYPE] = GEOGRAPHY_BINARY_TYPE_VALUE; + arrow_type = arrow::binary(); + break; + case iceberg::TypeID::VARIANT: arrow_type = arrow::extension::variant(arrow::struct_({ arrow::field("metadata", arrow::binary(), false), diff --git a/be/src/format/table/iceberg/arrow_schema_util.h b/be/src/format/table/iceberg/arrow_schema_util.h index 2942edacbaacf9..a8363f601c8720 100644 --- a/be/src/format/table/iceberg/arrow_schema_util.h +++ b/be/src/format/table/iceberg/arrow_schema_util.h @@ -34,6 +34,9 @@ class ArrowSchemaUtil { static const char* ORIGINAL_TYPE; static const char* MAP_TYPE_VALUE; static const char* UUID_TYPE_VALUE; + static const char* ICEBERG_BINARY_TYPE; + static const char* GEOMETRY_BINARY_TYPE_VALUE; + static const char* GEOGRAPHY_BINARY_TYPE_VALUE; static Status convert_to(const iceberg::NestedField& field, std::shared_ptr* arrow_field, diff --git a/be/src/format/table/iceberg/types.cpp b/be/src/format/table/iceberg/types.cpp index e9c9edb2900ec0..4693f4943bf1d1 100644 --- a/be/src/format/table/iceberg/types.cpp +++ b/be/src/format/table/iceberg/types.cpp @@ -143,6 +143,15 @@ std::string StructType::to_string() const { } std::unique_ptr Types::from_primitive_string(const std::string& type_string) { + auto trim = [](std::string value) { + const auto begin = value.find_first_not_of(" \t\n\r\f\v"); + if (begin == std::string::npos) { + return std::string {}; + } + const auto end = value.find_last_not_of(" \t\n\r\f\v"); + return value.substr(begin, end - begin + 1); + }; + std::string lower_type_string; std::transform(type_string.begin(), type_string.end(), std::back_inserter(lower_type_string), [](unsigned char c) { return std::tolower(c); }); @@ -189,6 +198,31 @@ std::unique_ptr Types::from_primitive_string(const std::string& t return std::make_unique(precision, scale); } + std::regex geometry(R"(geometry\s*(?:\(\s*([^)]*?)\s*\))?)", std::regex::icase); + if (std::regex_match(type_string, match, geometry, std::regex_constants::match_default)) { + const std::string crs = + match[1].matched ? trim(match[1].str()) : GeometryType::DEFAULT_CRS; + if (crs.empty()) { + throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, + "Invalid CRS: (empty string)"); + } + return std::make_unique(crs); + } + + std::regex geography(R"(geography\s*(?:\(\s*([^,]*?)\s*(?:,\s*(\w*)\s*)?\))?)", + std::regex::icase); + if (std::regex_match(type_string, match, geography, std::regex_constants::match_default)) { + const std::string crs = + match[1].matched ? trim(match[1].str()) : GeographyType::DEFAULT_CRS; + const std::string algorithm = + match[2].matched ? trim(match[2].str()) : GeographyType::DEFAULT_ALGORITHM; + if (crs.empty() || algorithm.empty()) { + throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, + "Spatial CRS and edge algorithm must not be empty"); + } + return std::make_unique(crs, algorithm); + } + throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, "Cannot parse type string to primitive: {}.", type_string); } diff --git a/be/src/format/table/iceberg/types.h b/be/src/format/table/iceberg/types.h index 6a54146dcd7dc6..c6bbfaf35444c4 100644 --- a/be/src/format/table/iceberg/types.h +++ b/be/src/format/table/iceberg/types.h @@ -50,7 +50,9 @@ enum TypeID { DECIMAL, STRUCT, LIST, - MAP + MAP, + GEOMETRY, + GEOGRAPHY }; class Type { @@ -278,6 +280,43 @@ class BinaryType : public PrimitiveType { std::string to_string() const override { return "binary"; } }; +class GeometryType : public PrimitiveType { +public: + static constexpr const char* DEFAULT_CRS = "OGC:CRS84"; + + explicit GeometryType(std::string crs = DEFAULT_CRS) : _crs(std::move(crs)) {} + + TypeID type_id() const override { return TypeID::GEOMETRY; } + + const std::string& crs() const { return _crs; } + + std::string to_string() const override { return "geometry(" + _crs + ")"; } + +private: + std::string _crs; +}; + +class GeographyType : public PrimitiveType { +public: + static constexpr const char* DEFAULT_CRS = "OGC:CRS84"; + static constexpr const char* DEFAULT_ALGORITHM = "spherical"; + + GeographyType(std::string crs = DEFAULT_CRS, std::string algorithm = DEFAULT_ALGORITHM) + : _crs(std::move(crs)), _algorithm(std::move(algorithm)) {} + + TypeID type_id() const override { return TypeID::GEOGRAPHY; } + + const std::string& crs() const { return _crs; } + + const std::string& algorithm() const { return _algorithm; } + + std::string to_string() const override { return "geography(" + _crs + ", " + _algorithm + ")"; } + +private: + std::string _crs; + std::string _algorithm; +}; + class VariantType : public PrimitiveType { public: ~VariantType() override = default; diff --git a/be/src/format/transformer/vparquet_transformer.cpp b/be/src/format/transformer/vparquet_transformer.cpp index 0900b85b94c94c..9af11fe7175f1e 100644 --- a/be/src/format/transformer/vparquet_transformer.cpp +++ b/be/src/format/transformer/vparquet_transformer.cpp @@ -23,7 +23,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -31,12 +33,14 @@ #include #include +#include #include #include #include #include "common/config.h" #include "common/status.h" +#include "exprs/function/geo/functions_geo.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format/arrow/arrow_block_convertor.h" @@ -59,6 +63,110 @@ arrow::MemoryPool* get_arrow_memory_pool() { return pool != nullptr ? pool : arrow::default_memory_pool(); } +::parquet::schema::NodePtr clone_parquet_node(const ::parquet::schema::Node& node) { + if (node.is_primitive()) { + const auto& primitive = static_cast(node); + return ::parquet::schema::PrimitiveNode::Make( + primitive.name(), primitive.repetition(), primitive.logical_type(), + primitive.physical_type(), primitive.type_length(), primitive.field_id()); + } + + const auto& group = static_cast(node); + ::parquet::schema::NodeVector fields; + fields.reserve(group.field_count()); + for (int index = 0; index < group.field_count(); ++index) { + fields.push_back(clone_parquet_node(*group.field(index))); + } + return ::parquet::schema::GroupNode::Make(group.name(), group.repetition(), fields, + group.logical_type(), group.field_id()); +} + +arrow::Result<::parquet::LogicalType::EdgeInterpolationAlgorithm> geography_algorithm( + const std::string& algorithm) { + using Algorithm = ::parquet::LogicalType::EdgeInterpolationAlgorithm; + if (algorithm == "spherical") { + return Algorithm::SPHERICAL; + } + if (algorithm == "vincenty") { + return Algorithm::VINCENTY; + } + if (algorithm == "thomas") { + return Algorithm::THOMAS; + } + if (algorithm == "andoyer") { + return Algorithm::ANDOYER; + } + if (algorithm == "karney") { + return Algorithm::KARNEY; + } + return arrow::Status::Invalid("Unsupported Iceberg geography edge algorithm: ", algorithm); +} + +bool contains_spatial_type(iceberg::Type* type) { + switch (type->type_id()) { + case iceberg::TypeID::GEOMETRY: + case iceberg::TypeID::GEOGRAPHY: + return true; + case iceberg::TypeID::STRUCT: + for (const auto& field : type->as_struct_type()->fields()) { + if (contains_spatial_type(field.field_type())) { + return true; + } + } + return false; + case iceberg::TypeID::LIST: + return contains_spatial_type(type->as_list_type()->element_field().field_type()); + case iceberg::TypeID::MAP: + return contains_spatial_type(type->as_map_type()->key_field().field_type()) || + contains_spatial_type(type->as_map_type()->value_field().field_type()); + default: + return false; + } +} + +arrow::Result> make_iceberg_parquet_schema( + const std::shared_ptr& arrow_schema, const iceberg::Schema& iceberg_schema, + const std::shared_ptr<::parquet::WriterProperties>& writer_properties, + const std::shared_ptr<::parquet::ArrowWriterProperties>& arrow_properties) { + std::shared_ptr<::parquet::SchemaDescriptor> parquet_schema; + ARROW_RETURN_NOT_OK(::parquet::arrow::ToParquetSchema(arrow_schema.get(), *writer_properties, + *arrow_properties, &parquet_schema)); + + const auto* source_root = parquet_schema->group_node(); + ::parquet::schema::NodeVector fields; + fields.reserve(source_root->field_count()); + for (int index = 0; index < source_root->field_count(); ++index) { + const auto& source_field = source_root->field(index); + const auto& iceberg_field = iceberg_schema.columns()[index]; + if (iceberg_field.field_type()->type_id() == iceberg::TypeID::GEOMETRY) { + const auto& geometry = + static_cast(*iceberg_field.field_type()); + fields.push_back(::parquet::schema::PrimitiveNode::Make( + source_field->name(), source_field->repetition(), + ::parquet::LogicalType::Geometry(geometry.crs()), ::parquet::Type::BYTE_ARRAY, + -1, source_field->field_id())); + } else if (iceberg_field.field_type()->type_id() == iceberg::TypeID::GEOGRAPHY) { + const auto& geography = + static_cast(*iceberg_field.field_type()); + ARROW_ASSIGN_OR_RAISE(auto algorithm, geography_algorithm(geography.algorithm())); + fields.push_back(::parquet::schema::PrimitiveNode::Make( + source_field->name(), source_field->repetition(), + ::parquet::LogicalType::Geography(geography.crs(), algorithm), + ::parquet::Type::BYTE_ARRAY, -1, source_field->field_id())); + } else if (contains_spatial_type(iceberg_field.field_type())) { + return arrow::Status::NotImplemented( + "Nested Iceberg spatial columns are not supported for Parquet writes: ", + iceberg_field.field_name()); + } else { + fields.push_back(clone_parquet_node(*source_field)); + } + } + return std::static_pointer_cast<::parquet::schema::GroupNode>( + ::parquet::schema::GroupNode::Make(source_root->name(), source_root->repetition(), + fields, source_root->logical_type(), + source_root->field_id())); +} + } // namespace ParquetOutputStream::ParquetOutputStream(doris::io::FileWriter* file_writer) @@ -279,6 +387,12 @@ Status VParquetTransformer::write(const Block& block) { return Status::OK(); } + if (_iceberg_schema != nullptr) { + ColumnNumbers column_numbers(block.columns()); + std::iota(column_numbers.begin(), column_numbers.end(), 0); + RETURN_IF_ERROR(validate_spatial_wkb_inputs(block, column_numbers)); + } + // serialize std::shared_ptr result; RETURN_IF_ERROR(convert_to_arrow_batch(block, _arrow_schema, get_arrow_memory_pool(), &result, @@ -295,6 +409,24 @@ Status VParquetTransformer::write(const Block& block) { } arrow::Status VParquetTransformer::_open_file_writer() { + if (_iceberg_schema != nullptr) { + bool has_spatial_column = false; + for (const auto& column : _iceberg_schema->columns()) { + has_spatial_column = has_spatial_column || contains_spatial_type(column.field_type()); + } + if (has_spatial_column) { + ARROW_ASSIGN_OR_RAISE(auto parquet_schema, + make_iceberg_parquet_schema(_arrow_schema, *_iceberg_schema, + _parquet_writer_properties, + _arrow_properties)); + auto parquet_writer = ::parquet::ParquetFileWriter::Open( + _outstream, std::move(parquet_schema), _parquet_writer_properties); + ARROW_RETURN_NOT_OK(::parquet::arrow::FileWriter::Make( + get_arrow_memory_pool(), std::move(parquet_writer), _arrow_schema, + _arrow_properties, &_writer)); + return arrow::Status::OK(); + } + } ARROW_ASSIGN_OR_RAISE(_writer, ::parquet::arrow::FileWriter::Open( *_arrow_schema, get_arrow_memory_pool(), _outstream, _parquet_writer_properties, _arrow_properties)); diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index f9da3adfbefe80..d670d80cc945fe 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -31,6 +31,7 @@ #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_spatial.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" #include "util/slice.h" @@ -61,6 +62,23 @@ static bool is_variant_node(const tparquet::SchemaElement& schema) { return schema.__isset.logicalType && schema.logicalType.__isset.VARIANT; } +static std::string geography_algorithm_to_string( + tparquet::EdgeInterpolationAlgorithm::type algorithm) { + switch (algorithm) { + case tparquet::EdgeInterpolationAlgorithm::SPHERICAL: + return "spherical"; + case tparquet::EdgeInterpolationAlgorithm::VINCENTY: + return "vincenty"; + case tparquet::EdgeInterpolationAlgorithm::THOMAS: + return "thomas"; + case tparquet::EdgeInterpolationAlgorithm::ANDOYER: + return "andoyer"; + case tparquet::EdgeInterpolationAlgorithm::KARNEY: + return "karney"; + } + throw Exception(Status::InternalError("Unsupported Parquet geography edge algorithm")); +} + enum class VariantPrimitiveAnnotation : uint8_t { NONE, INT8, @@ -907,6 +925,16 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( } } else if (logicalType.__isset.FLOAT16) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + } else if (logicalType.__isset.GEOMETRY) { + const auto& geometry = logicalType.GEOMETRY; + ans.first = std::make_shared( + TYPE_GEOMETRY, geometry.__isset.crs ? geometry.crs : "OGC:CRS84"); + } else if (logicalType.__isset.GEOGRAPHY) { + const auto& geography = logicalType.GEOGRAPHY; + ans.first = std::make_shared( + TYPE_GEOGRAPHY, geography.__isset.crs ? geography.crs : "OGC:CRS84", + geography.__isset.algorithm ? geography_algorithm_to_string(geography.algorithm) + : "spherical"); } else { throw Exception(Status::InternalError("Not supported parquet logicalType")); } diff --git a/be/test/core/data_type/data_type_spatial_test.cpp b/be/test/core/data_type/data_type_spatial_test.cpp new file mode 100644 index 00000000000000..65529d9fa32017 --- /dev/null +++ b/be/test/core/data_type/data_type_spatial_test.cpp @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_spatial.h" + +#include +#include +#include + +#include +#include + +#include "agent/be_exec_version_manager.h" +#include "core/assert_cast.h" +#include "core/column/column_spatial.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_varbinary.h" +#include "exec/common/arrow_column_to_doris_column.h" + +namespace doris { + +TEST(DataTypeSpatialTest, FactoryCreatesDistinctSpatialColumns) { + auto geometry = DataTypeFactory::instance().create_data_type(TYPE_GEOMETRY, false); + auto geography = DataTypeFactory::instance().create_data_type(TYPE_GEOGRAPHY, false); + + ASSERT_NE(nullptr, geometry); + ASSERT_NE(nullptr, geography); + EXPECT_EQ(TYPE_GEOMETRY, geometry->get_primitive_type()); + EXPECT_EQ(TYPE_GEOGRAPHY, geography->get_primitive_type()); + EXPECT_FALSE(geometry->equals(*geography)); + + auto geometry_column = geometry->create_column(); + auto geography_column = geography->create_column(); + EXPECT_TRUE(geometry->check_column(*geometry_column).ok()); + EXPECT_TRUE(geography->check_column(*geography_column).ok()); + EXPECT_FALSE(geometry->check_column(*geography_column).ok()); + EXPECT_FALSE(geometry->check_column(*ColumnVarbinary::create()).ok()); + + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00@", 21); + auto& column = assert_cast(*geometry_column); + column.insert_data(wkb.data(), wkb.size()); + + const auto value = column.get_data_at(0); + EXPECT_EQ(wkb.size(), value.size); + EXPECT_EQ(0, memcmp(wkb.data(), value.data, value.size)); + + const auto version = BeExecVersionManager::get_newest_version(); + const auto bytes = geometry->get_uncompressed_serialized_bytes(*geometry_column, version); + std::string buffer(bytes, '\0'); + auto* end = geometry->serialize(*geometry_column, buffer.data(), version); + EXPECT_EQ(buffer.data() + buffer.size(), end); + + auto restored = geometry->create_column(); + const auto* restored_end = geometry->deserialize(buffer.data(), &restored, version); + EXPECT_EQ(buffer.data() + buffer.size(), restored_end); + const auto restored_value = restored->get_data_at(0); + EXPECT_EQ(wkb.size(), restored_value.size); + EXPECT_EQ(0, memcmp(wkb.data(), restored_value.data, restored_value.size)); +} + +TEST(DataTypeSpatialTest, FactoryPreservesSpatialMetadataFromTypeDescriptor) { + auto geometry_desc = create_type_desc(TYPE_GEOMETRY); + geometry_desc.types[0].scalar_type.__set_spatial_crs("EPSG:3857"); + const auto geometry = DataTypeFactory::instance().create_data_type(geometry_desc); + const auto& geometry_type = assert_cast(*geometry); + EXPECT_EQ(TYPE_GEOMETRY, geometry_type.get_primitive_type()); + EXPECT_EQ("EPSG:3857", geometry_type.crs()); + EXPECT_TRUE(geometry_type.algorithm().empty()); + + auto geography_desc = create_type_desc(TYPE_GEOGRAPHY); + geography_desc.types[0].scalar_type.__set_spatial_crs("OGC:CRS84"); + geography_desc.types[0].scalar_type.__set_spatial_algorithm("spherical"); + const auto geography = DataTypeFactory::instance().create_data_type(geography_desc); + const auto& geography_type = assert_cast(*geography); + EXPECT_EQ(TYPE_GEOGRAPHY, geography_type.get_primitive_type()); + EXPECT_EQ("OGC:CRS84", geography_type.crs()); + EXPECT_EQ("spherical", geography_type.algorithm()); +} + +TEST(DataTypeSpatialTest, ProtobufRoundTripPreservesSpatialMetadata) { + const auto geography = std::make_shared(TYPE_GEOGRAPHY, "EPSG:4326", "vincenty"); + PTypeDesc descriptor; + static_cast(*geography).to_protobuf(&descriptor); + + ASSERT_EQ(1, descriptor.types_size()); + ASSERT_TRUE(descriptor.types(0).has_spatial_crs()); + ASSERT_TRUE(descriptor.types(0).has_spatial_algorithm()); + EXPECT_EQ("EPSG:4326", descriptor.types(0).spatial_crs()); + EXPECT_EQ("vincenty", descriptor.types(0).spatial_algorithm()); + + int index = 0; + const auto restored = DataTypeFactory::instance().create_data_type(descriptor.types(), &index, false); + const auto& restored_type = assert_cast(*restored); + EXPECT_EQ(TYPE_GEOGRAPHY, restored_type.get_primitive_type()); + EXPECT_EQ("EPSG:4326", restored_type.crs()); + EXPECT_EQ("vincenty", restored_type.algorithm()); +} + +TEST(DataTypeSpatialTest, ArrowBinaryReadPreservesWkbForSpatialTypes) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00@", 21); + arrow::BinaryBuilder builder; + ASSERT_TRUE(builder.Append(wkb).ok()); + std::shared_ptr arrow_array; + ASSERT_TRUE(builder.Finish(&arrow_array).ok()); + + for (const auto primitive_type : {TYPE_GEOMETRY, TYPE_GEOGRAPHY}) { + const auto type = DataTypeFactory::instance().create_data_type(primitive_type, false); + ColumnPtr column = type->create_column(); + ASSERT_TRUE(arrow_column_to_doris_column(arrow_array.get(), 0, column, type, 1, "").ok()); + + const auto& spatial = assert_cast(*column); + EXPECT_EQ(primitive_type, spatial.get_primitive_type()); + ASSERT_EQ(1, spatial.size()); + const auto value = spatial.get_data_at(0); + EXPECT_EQ(wkb.size(), value.size); + EXPECT_EQ(0, memcmp(wkb.data(), value.data, value.size)); + } +} + +TEST(DataTypeSpatialTest, ArrowBinaryWritePreservesWkbForSpatialTypes) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + const NullMap null_map = {0, 1}; + cctz::time_zone timezone; + + for (const auto primitive_type : {TYPE_GEOMETRY, TYPE_GEOGRAPHY}) { + const auto type = DataTypeFactory::instance().create_data_type(primitive_type, false); + auto column = type->create_column(); + column->insert_data(wkb.data(), wkb.size()); + column->insert_data(wkb.data(), wkb.size()); + + arrow::BinaryBuilder builder; + ASSERT_TRUE(type->get_serde() + ->write_column_to_arrow(*column, &null_map, &builder, 0, column->size(), + timezone) + .ok()); + + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + const auto* binary = dynamic_cast(array.get()); + ASSERT_NE(nullptr, binary); + ASSERT_EQ(2, binary->length()); + ASSERT_FALSE(binary->IsNull(0)); + ASSERT_TRUE(binary->IsNull(1)); + ASSERT_EQ(wkb.size(), static_cast(binary->value_length(0))); + const auto* value = binary->value_data()->data() + binary->value_offset(0); + EXPECT_EQ(0, memcmp(wkb.data(), value, wkb.size())); + } +} + +TEST(DataTypeSpatialTest, ColumnPreservesWkbThroughFilterAndPermute) { + auto column = ColumnSpatial::create(TYPE_GEOMETRY); + const std::string first("\x01\x01\x00\x00\x00", 5); + const std::string second("\x00\x02\x00\x00\x00\x10\x00", 7); + const std::string third("\x01\x03\x00\x00\x00\x00", 6); + column->insert_data(first.data(), first.size()); + column->insert_data(second.data(), second.size()); + column->insert_data(third.data(), third.size()); + + const IColumn::Filter filter = {1, 0, 1}; + const auto filtered = column->filter(filter, -1); + ASSERT_EQ(2, filtered->size()); + EXPECT_EQ(first, filtered->get_data_at(0).to_string()); + EXPECT_EQ(third, filtered->get_data_at(1).to_string()); + + const IColumn::Permutation permutation = {2, 0, 1}; + const auto permuted = column->permute(permutation, 2); + ASSERT_EQ(2, permuted->size()); + EXPECT_EQ(third, permuted->get_data_at(0).to_string()); + EXPECT_EQ(first, permuted->get_data_at(1).to_string()); +} + +} // namespace doris diff --git a/be/test/core/data_type/storage_field_type_test.cpp b/be/test/core/data_type/storage_field_type_test.cpp index af621ceb9c6abe..3eeffa9bb40370 100644 --- a/be/test/core/data_type/storage_field_type_test.cpp +++ b/be/test/core/data_type/storage_field_type_test.cpp @@ -90,6 +90,8 @@ TEST(StorageFieldTypeTest, UnsupportedPrimitiveTypesThrow) { static_cast(33), // TYPE_LAMBDA_FUNCTION (deprecated) PrimitiveType::TYPE_FIXED_LENGTH_OBJECT, PrimitiveType::TYPE_VARBINARY, + PrimitiveType::TYPE_GEOMETRY, + PrimitiveType::TYPE_GEOGRAPHY, static_cast(43), static_cast(255), }; diff --git a/be/test/exprs/function/function_geo_test.cpp b/be/test/exprs/function/function_geo_test.cpp index b4993a231e88c8..b5595256c2c4e3 100644 --- a/be/test/exprs/function/function_geo_test.cpp +++ b/be/test/exprs/function/function_geo_test.cpp @@ -25,8 +25,11 @@ #include #include "common/status.h" +#include "core/assert_cast.h" +#include "core/column/column_spatial.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_spatial.h" #include "core/data_type/data_type_string.h" #include "core/types.h" #include "exprs/function/function_test_util.h" @@ -94,6 +97,476 @@ TEST(VGeoFunctionsTest, function_geo_st_as_text) { } } +TEST(VGeoFunctionsTest, function_geo_st_as_text_with_spatial_wkb) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + for (const auto primitive_type : {TYPE_GEOMETRY, TYPE_GEOGRAPHY}) { + auto spatial_type = primitive_type == TYPE_GEOMETRY + ? std::make_shared(TYPE_GEOMETRY) + : std::make_shared(TYPE_GEOGRAPHY, "OGC:CRS84", + "spherical"); + auto spatial_column = ColumnSpatial::create(primitive_type); + spatial_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(spatial_column), spatial_type, "spatial"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_astext", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + + const auto value = block.get_by_position(1).column->get_data_at(0); + EXPECT_EQ("POINT (1 2)", std::string(value.data, value.size)); + } +} + +TEST(VGeoFunctionsTest, function_geo_st_as_text_preserves_null_spatial_rows) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + auto spatial_values = ColumnSpatial::create(TYPE_GEOMETRY); + spatial_values->insert_data(wkb.data(), wkb.size()); + spatial_values->insert_default(); + auto null_map = ColumnUInt8::create(); + null_map->insert_value(0); + null_map->insert_value(1); + + auto spatial_type = make_nullable(std::make_shared(TYPE_GEOMETRY)); + ColumnsWithTypeAndName arguments { + {ColumnNullable::create(std::move(spatial_values), std::move(null_map)), spatial_type, + "spatial"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_astext", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 2).ok()); + + const auto& result = block.get_by_position(1).column; + EXPECT_EQ("POINT (1 2)", std::string(result->get_data_at(0).data, result->get_data_at(0).size)); + EXPECT_TRUE(result->is_null_at(1)); +} + +TEST(VGeoFunctionsTest, function_geo_st_geometryfromwkbtyped_returns_raw_geometry_wkb) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + for (const auto& function_name : {"st_geometryfromwkbtyped"}) { + auto input_column = ColumnString::create(); + input_column->insert_data("0101000000000000000000F03F0000000000000040", 42); + auto input_type = std::make_shared(); + ColumnsWithTypeAndName arguments {{std::move(input_column), input_type, "wkb"}}; + auto result_type = make_nullable(std::make_shared(TYPE_GEOMETRY)); + auto function = SimpleFunctionFactory::instance().get_function(function_name, arguments, + result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + + const auto value = block.get_by_position(1).column->get_data_at(0); + EXPECT_EQ(wkb, std::string(value.data, value.size)); + } +} + +TEST(VGeoFunctionsTest, function_geo_st_geometryfromwkbtyped_accepts_0x_prefixed_wkb) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + auto input_column = ColumnString::create(); + input_column->insert_data("0x0101000000000000000000F03F0000000000000040", 44); + auto input_type = std::make_shared(); + ColumnsWithTypeAndName arguments {{std::move(input_column), input_type, "wkb"}}; + auto result_type = make_nullable(std::make_shared(TYPE_GEOMETRY)); + auto function = SimpleFunctionFactory::instance().get_function("st_geometryfromwkbtyped", + arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + + const auto value = block.get_by_position(1).column->get_data_at(0); + EXPECT_EQ(wkb, std::string(value.data, value.size)); +} + +TEST(VGeoFunctionsTest, function_geo_st_geogfromwkb_returns_raw_geography_wkb) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + auto input_column = ColumnString::create(); + input_column->insert_data("0101000000000000000000F03F0000000000000040", 42); + auto input_type = std::make_shared(); + ColumnsWithTypeAndName arguments {{std::move(input_column), input_type, "wkb"}}; + auto result_type = make_nullable( + std::make_shared(TYPE_GEOGRAPHY, "OGC:CRS84", "spherical")); + auto function = SimpleFunctionFactory::instance().get_function("st_geogfromwkb", arguments, + result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + + EXPECT_EQ(TYPE_GEOGRAPHY, remove_nullable(block.get_data_type(1))->get_primitive_type()); + const auto value = block.get_by_position(1).column->get_data_at(0); + EXPECT_EQ(wkb, std::string(value.data, value.size)); +} + +TEST(VGeoFunctionsTest, function_geo_fromwkb_rejects_unsupported_metadata) { + const std::vector unsupported_wkb { + "0101000080000000000000F03F00000000000000400000000000000840", + "0101000020E6100000000000000000F03F0000000000000040"}; + for (const auto& function_name : {"st_geometryfromwkbtyped", "st_geogfromwkb"}) { + for (const auto& wkb : unsupported_wkb) { + auto input_column = ColumnString::create(); + input_column->insert_data(wkb.data(), wkb.size()); + auto input_type = std::make_shared(); + ColumnsWithTypeAndName arguments {{std::move(input_column), input_type, "wkb"}}; + auto result_type = + function_name == "st_geometryfromwkbtyped" + ? make_nullable(std::make_shared(TYPE_GEOMETRY)) + : make_nullable(std::make_shared( + TYPE_GEOGRAPHY, "OGC:CRS84", "spherical")); + auto function = SimpleFunctionFactory::instance().get_function(function_name, arguments, + result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + EXPECT_TRUE(block.get_by_position(1).column->is_null_at(0)); + } + } +} + +TEST(VGeoFunctionsTest, function_geo_st_distance_rejects_geometry) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + auto geometry_type = std::make_shared(TYPE_GEOMETRY); + auto left_column = ColumnSpatial::create(TYPE_GEOMETRY); + auto right_column = ColumnSpatial::create(TYPE_GEOMETRY); + left_column->insert_data(wkb.data(), wkb.size()); + right_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(left_column), geometry_type, "left"}, + {std::move(right_column), geometry_type, "right"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_distance", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments[0]); + block.insert(arguments[1]); + block.insert({nullptr, result_type, "result"}); + const auto status = function->execute(nullptr, block, {0, 1}, 2, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("GEOGRAPHY(OGC:CRS84, spherical)"), std::string::npos) + << status.to_string(); +} + +TEST(VGeoFunctionsTest, function_geo_st_distance_accepts_supported_geography) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + auto geography_type = + std::make_shared(TYPE_GEOGRAPHY, "OGC:CRS84", "spherical"); + auto left_column = ColumnSpatial::create(TYPE_GEOGRAPHY); + auto right_column = ColumnSpatial::create(TYPE_GEOGRAPHY); + left_column->insert_data(wkb.data(), wkb.size()); + right_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(left_column), geography_type, "left"}, + {std::move(right_column), geography_type, "right"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_distance", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments[0]); + block.insert(arguments[1]); + block.insert({nullptr, result_type, "result"}); + EXPECT_TRUE(function->execute(nullptr, block, {0, 1}, 2, 1).ok()); +} + +TEST(VGeoFunctionsTest, function_geo_st_distance_rejects_unsupported_geography_metadata) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + for (const auto& geography_type : std::vector { + std::make_shared(TYPE_GEOGRAPHY, "EPSG:4326", "spherical"), + std::make_shared(TYPE_GEOGRAPHY, "OGC:CRS84", "vincenty")}) { + auto left_column = ColumnSpatial::create(TYPE_GEOGRAPHY); + auto right_column = ColumnSpatial::create(TYPE_GEOGRAPHY); + left_column->insert_data(wkb.data(), wkb.size()); + right_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(left_column), geography_type, "left"}, + {std::move(right_column), geography_type, "right"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = SimpleFunctionFactory::instance().get_function("st_distance", arguments, + result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments[0]); + block.insert(arguments[1]); + block.insert({nullptr, result_type, "result"}); + const auto status = function->execute(nullptr, block, {0, 1}, 2, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("GEOGRAPHY(OGC:CRS84, spherical)"), std::string::npos) + << status.to_string(); + } +} + +TEST(VGeoFunctionsTest, function_geo_st_contains_rejects_geometry) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + auto geometry_type = std::make_shared(TYPE_GEOMETRY); + auto left_column = ColumnSpatial::create(TYPE_GEOMETRY); + auto right_column = ColumnSpatial::create(TYPE_GEOMETRY); + left_column->insert_data(wkb.data(), wkb.size()); + right_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(left_column), geometry_type, "left"}, + {std::move(right_column), geometry_type, "right"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_contains", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments[0]); + block.insert(arguments[1]); + block.insert({nullptr, result_type, "result"}); + const auto status = function->execute(nullptr, block, {0, 1}, 2, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("GEOGRAPHY(OGC:CRS84, spherical)"), std::string::npos) + << status.to_string(); +} + +TEST(VGeoFunctionsTest, function_geo_st_astext_rejects_invalid_spatial_wkb) { + auto geometry_type = std::make_shared(TYPE_GEOMETRY); + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data("\x01", 1); + + ColumnsWithTypeAndName arguments {{std::move(geometry_column), geometry_type, "geometry"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_astext", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + const auto status = function->execute(nullptr, block, {0}, 1, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid WKB in spatial input"), std::string::npos) + << status.to_string(); + EXPECT_NE(status.to_string().find("WKB syntax error"), std::string::npos) << status.to_string(); +} + +TEST(VGeoFunctionsTest, function_geo_st_astext_rejects_spatial_wkb_with_trailing_bytes) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@\x00", + 22); + auto geometry_type = std::make_shared(TYPE_GEOMETRY); + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(geometry_column), geometry_type, "geometry"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_astext", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + const auto status = function->execute(nullptr, block, {0}, 1, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Invalid WKB in spatial input"), std::string::npos) + << status.to_string(); +} + +TEST(VGeoFunctionsTest, function_geo_st_astext_rejects_spatial_wkb_with_z) { + const std::string point_z_wkb( + "\x01\x01\x00\x00\x80" + "\x00\x00\x00\x00\x00\x00\xf0?" + "\x00\x00\x00\x00\x00\x00\x00@" + "\x00\x00\x00\x00\x00\x00\x08@", + 29); + auto geometry_type = std::make_shared(TYPE_GEOMETRY); + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data(point_z_wkb.data(), point_z_wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(geometry_column), geometry_type, "geometry"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_astext", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + const auto status = function->execute(nullptr, block, {0}, 1, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("WKB dimensions or embedded SRID are not supported"), + std::string::npos) + << status.to_string(); +} + +TEST(VGeoFunctionsTest, function_geo_st_astext_rejects_spatial_wkb_with_m_or_srid) { + const std::vector unsupported_wkb { + std::string("\x01\x01\x00\x00\x40" + "\x00\x00\x00\x00\x00\x00\xf0?" + "\x00\x00\x00\x00\x00\x00\x00@" + "\x00\x00\x00\x00\x00\x00\x08@", + 29), + std::string("\x01\x01\x00\x00\x20\xe6\x10\x00\x00" + "\x00\x00\x00\x00\x00\x00\xf0?" + "\x00\x00\x00\x00\x00\x00\x00@", + 25)}; + + for (const auto& wkb : unsupported_wkb) { + auto geometry_type = std::make_shared(TYPE_GEOMETRY); + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(geometry_column), geometry_type, "geometry"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = + SimpleFunctionFactory::instance().get_function("st_astext", arguments, result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + const auto status = function->execute(nullptr, block, {0}, 1, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("WKB dimensions or embedded SRID are not supported"), + std::string::npos) + << status.to_string(); + } +} + +TEST(VGeoFunctionsTest, function_geo_point_accessors_with_spatial_wkb) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@", 21); + for (const auto& [function_name, expected] : + std::vector> {{"st_x", 1.0}, {"st_y", 2.0}}) { + auto geometry_type = std::make_shared(TYPE_GEOMETRY); + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data(wkb.data(), wkb.size()); + + ColumnsWithTypeAndName arguments {{std::move(geometry_column), geometry_type, "geometry"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = SimpleFunctionFactory::instance().get_function(function_name, arguments, + result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + + const auto& result = assert_cast(*block.get_by_position(1).column); + EXPECT_EQ(0, result.get_null_map_data()[0]); + const auto& values = assert_cast(result.get_nested_column()); + EXPECT_DOUBLE_EQ(expected, values.get_data()[0]); + } +} + +TEST(VGeoFunctionsTest, function_geo_projected_geometry_accessors) { + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00@\x8f@\x00\x00\x00\x00\x00@\x9f@", 21); + const auto geometry_type = std::make_shared(TYPE_GEOMETRY, "EPSG:3857"); + for (const auto& [function_name, expected] : + std::vector> {{"st_astext", "POINT (1000 2000)"}, + {"st_geometrytype", "ST_POINT"}, + {"st_asbinary", wkb}}) { + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data(wkb.data(), wkb.size()); + ColumnsWithTypeAndName arguments {{std::move(geometry_column), geometry_type, "geometry"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = SimpleFunctionFactory::instance().get_function(function_name, arguments, + result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + const auto value = block.get_by_position(1).column->get_data_at(0); + EXPECT_EQ(expected, std::string(value.data, value.size)); + } + + for (const auto& [function_name, expected] : + std::vector> {{"st_x", 1000.0}, {"st_y", 2000.0}}) { + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data(wkb.data(), wkb.size()); + ColumnsWithTypeAndName arguments {{std::move(geometry_column), geometry_type, "geometry"}}; + auto result_type = make_nullable(std::make_shared()); + auto function = SimpleFunctionFactory::instance().get_function(function_name, arguments, + result_type); + ASSERT_NE(nullptr, function); + + Block block; + block.insert(arguments.front()); + block.insert({nullptr, result_type, "result"}); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 1).ok()); + const auto& result = assert_cast(*block.get_by_position(1).column); + EXPECT_EQ(0, result.get_null_map_data()[0]); + EXPECT_DOUBLE_EQ( + expected, + assert_cast(result.get_nested_column()).get_data()[0]); + } +} + +TEST(VGeoFunctionsTest, function_geo_legacy_wkb_accepts_embedded_srid) { + const std::string ewkb = "01010000208A11000068270210774C5D40B8DECA334C3B4240"; + auto input_column = ColumnString::create(); + input_column->insert_data(ewkb.data(), ewkb.size()); + auto input_type = std::make_shared(); + ColumnsWithTypeAndName arguments {{std::move(input_column), input_type, "wkb"}}; + auto legacy_result_type = make_nullable(std::make_shared()); + auto legacy = SimpleFunctionFactory::instance().get_function("st_geometryfromwkb", arguments, + legacy_result_type); + ASSERT_NE(nullptr, legacy); + + Block legacy_block; + legacy_block.insert(arguments.front()); + legacy_block.insert({nullptr, legacy_result_type, "result"}); + ASSERT_TRUE(legacy->execute(nullptr, legacy_block, {0}, 1, 1).ok()); + const auto& legacy_result = legacy_block.get_by_position(1).column; + EXPECT_FALSE(legacy_result->is_null_at(0)); + const auto encoded = legacy_result->get_data_at(0); + const auto shape = GeoShape::from_encoded(encoded.data, encoded.size); + ASSERT_NE(nullptr, shape); + EXPECT_EQ("POINT (81.194767 36.462286)", shape->as_wkt()); + + auto typed_column = ColumnString::create(); + typed_column->insert_data(ewkb.data(), ewkb.size()); + ColumnsWithTypeAndName typed_arguments {{std::move(typed_column), input_type, "wkb"}}; + auto typed_result_type = make_nullable(std::make_shared(TYPE_GEOMETRY)); + auto typed = SimpleFunctionFactory::instance().get_function("st_geometryfromwkbtyped", + typed_arguments, typed_result_type); + ASSERT_NE(nullptr, typed); + Block typed_block; + typed_block.insert(typed_arguments.front()); + typed_block.insert({nullptr, typed_result_type, "result"}); + ASSERT_TRUE(typed->execute(nullptr, typed_block, {0}, 1, 1).ok()); + EXPECT_TRUE(typed_block.get_by_position(1).column->is_null_at(0)); +} + TEST(VGeoFunctionsTest, function_geo_st_as_wkt) { std::string func_name = "st_aswkt"; { diff --git a/be/test/format/table/iceberg/arrow_schema_util_test.cpp b/be/test/format/table/iceberg/arrow_schema_util_test.cpp index d824181263989c..a00463da49a508 100644 --- a/be/test/format/table/iceberg/arrow_schema_util_test.cpp +++ b/be/test/format/table/iceberg/arrow_schema_util_test.cpp @@ -254,6 +254,35 @@ TEST(ArrowSchemaUtilTest, test_binary_field_types) { EXPECT_EQ("uuid", fields[3]->metadata()->Get("originalType").ValueUnsafe()); } +TEST(ArrowSchemaUtilTest, test_geospatial_field_types) { + const std::string schema_json = R"JSON({ + "type": "struct", + "fields": [ + {"id": 101, "name": "shape", "required": false, "type": "GEOMETRY(EPSG:3857)"}, + {"id": 102, "name": "place", "required": false, + "type": "geography(OGC:CRS84, vincenty)"} + ] + })JSON"; + const auto schema = SchemaParser::from_json(schema_json); + const auto* geometry = static_cast(schema->columns()[0].field_type()); + const auto* geography = static_cast(schema->columns()[1].field_type()); + EXPECT_EQ("EPSG:3857", geometry->crs()); + EXPECT_EQ("OGC:CRS84", geography->crs()); + EXPECT_EQ("vincenty", geography->algorithm()); + + std::vector> fields; + ASSERT_TRUE(ArrowSchemaUtil::convert(schema.get(), "utc", fields).ok()); + ASSERT_EQ(2, fields.size()); + + EXPECT_EQ(arrow::Type::BINARY, fields[0]->type()->id()); + EXPECT_EQ("101", fields[0]->metadata()->Get(pfid).ValueUnsafe()); + EXPECT_EQ("GEOMETRY", fields[0]->metadata()->Get("iceberg.binary-type").ValueUnsafe()); + + EXPECT_EQ(arrow::Type::BINARY, fields[1]->type()->id()); + EXPECT_EQ("102", fields[1]->metadata()->Get(pfid).ValueUnsafe()); + EXPECT_EQ("GEOGRAPHY", fields[1]->metadata()->Get("iceberg.binary-type").ValueUnsafe()); +} + TEST(ArrowSchemaUtilTest, test_variant_field) { std::vector nested_fields; nested_fields.emplace_back(true, 21, "payload", std::make_unique(), std::nullopt); diff --git a/be/test/format/transformer/vparquet_transformer_test.cpp b/be/test/format/transformer/vparquet_transformer_test.cpp index 9a93adb7c55d3d..68754530beb45c 100644 --- a/be/test/format/transformer/vparquet_transformer_test.cpp +++ b/be/test/format/transformer/vparquet_transformer_test.cpp @@ -17,8 +17,13 @@ #include "format/transformer/vparquet_transformer.h" +#include +#include +#include +#include #include #include +#include #include #include @@ -26,9 +31,11 @@ #include "core/block/block.h" #include "core/column/column_array.h" #include "core/column/column_nullable.h" +#include "core/column/column_spatial.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_spatial.h" #include "core/data_type/data_type_variant_v2.h" #include "exprs/function/parse/variant_string_parse.h" #include "format/table/iceberg/schema_parser.h" @@ -128,6 +135,154 @@ TEST_F(VParquetTransformerTest, WritesIcebergVariantAndCollectsLogicalMetrics) { EXPECT_EQ(-1, payload_group.field(1)->field_id()); } +TEST_F(VParquetTransformerTest, WritesIcebergSpatialWkbAsBinary) { + auto geometry_type = std::make_shared(TYPE_GEOMETRY, "EPSG:3857"); + auto geography_type = + std::make_shared(TYPE_GEOGRAPHY, "OGC:CRS84", "vincenty"); + VExprContextSPtrs output_exprs = + MockSlotRef::create_mock_contexts(DataTypes {geometry_type, geography_type}); + + const std::string schema_json = R"JSON({ + "type": "struct", + "fields": [ + {"id": 1, "name": "shape", "required": false, "type": "geometry(EPSG:3857)"}, + {"id": 2, "name": "place", "required": false, + "type": "geography(OGC:CRS84, vincenty)"} + ] + })JSON"; + const auto schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("UTC"); + ParquetFileOptions options {.compression_type = TParquetCompressionType::UNCOMPRESSED, + .parquet_version = TParquetVersion::PARQUET_1_0, + .parquet_disable_dictionary = false, + .enable_int96_timestamps = false}; + VParquetTransformer transformer(&state, file_writer.get(), output_exprs, {"shape", "place"}, + false, options, &schema_json, schema.get()); + ASSERT_TRUE(transformer.open().ok()); + + const std::string wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00@", 21); + const std::string projected_wkb( + "\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00@" + "\x8f@\x00\x00\x00\x00\x00@\x9f@", + 21); + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + auto geography_column = ColumnSpatial::create(TYPE_GEOGRAPHY); + geometry_column->insert_data(projected_wkb.data(), projected_wkb.size()); + geography_column->insert_data(wkb.data(), wkb.size()); + Block block; + block.insert(ColumnWithTypeAndName(std::move(geometry_column), geometry_type, "shape")); + block.insert(ColumnWithTypeAndName(std::move(geography_column), geography_type, "place")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + const auto* root = reader->metadata()->schema()->group_node(); + ASSERT_EQ(2, root->field_count()); + const auto& shape = static_cast(*root->field(0)); + const auto& place = static_cast(*root->field(1)); + EXPECT_EQ(::parquet::Type::BYTE_ARRAY, shape.physical_type()); + EXPECT_TRUE(shape.logical_type()->is_geometry()); + EXPECT_EQ("EPSG:3857", + static_cast(*shape.logical_type()).crs()); + EXPECT_EQ(1, shape.field_id()); + EXPECT_EQ(::parquet::Type::BYTE_ARRAY, place.physical_type()); + EXPECT_TRUE(place.logical_type()->is_geography()); + const auto& geography_logical = + static_cast(*place.logical_type()); + EXPECT_EQ("OGC:CRS84", geography_logical.crs()); + EXPECT_EQ(::parquet::LogicalType::EdgeInterpolationAlgorithm::VINCENTY, + geography_logical.algorithm()); + EXPECT_EQ(2, place.field_id()); + + auto input_result = ::arrow::io::ReadableFile::Open(_file_path); + ASSERT_TRUE(input_result.ok()) << input_result.status(); + std::shared_ptr<::arrow::io::RandomAccessFile> input = std::move(input_result).ValueUnsafe(); + auto arrow_reader_result = ::parquet::arrow::OpenFile(input, ::arrow::default_memory_pool()); + ASSERT_TRUE(arrow_reader_result.ok()) << arrow_reader_result.status(); + auto arrow_reader = std::move(arrow_reader_result).ValueUnsafe(); + auto table_result = arrow_reader->ReadTable(); + ASSERT_TRUE(table_result.ok()) << table_result.status(); + const auto table = std::move(table_result).ValueUnsafe(); + + for (int column_index = 0; column_index < 2; ++column_index) { + const auto array = std::static_pointer_cast<::arrow::BinaryArray>( + table->column(column_index)->chunk(0)); + ASSERT_EQ(1, array->length()); + ASSERT_FALSE(array->IsNull(0)); + EXPECT_EQ(column_index == 0 ? projected_wkb : wkb, array->GetString(0)); + } + EXPECT_EQ("GEOMETRY", + table->schema()->field(0)->metadata()->Get("iceberg.binary-type").ValueUnsafe()); + EXPECT_EQ("GEOGRAPHY", + table->schema()->field(1)->metadata()->Get("iceberg.binary-type").ValueUnsafe()); +} + +TEST_F(VParquetTransformerTest, RejectsInvalidIcebergSpatialWkb) { + auto geometry_type = std::make_shared(TYPE_GEOMETRY, "EPSG:3857"); + VExprContextSPtrs output_exprs = MockSlotRef::create_mock_contexts(DataTypes {geometry_type}); + + const std::string schema_json = R"JSON({ + "type": "struct", + "fields": [ + {"id": 1, "name": "shape", "required": false, "type": "geometry(EPSG:3857)"} + ] + })JSON"; + const auto schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("UTC"); + ParquetFileOptions options {.compression_type = TParquetCompressionType::UNCOMPRESSED, + .parquet_version = TParquetVersion::PARQUET_1_0, + .parquet_disable_dictionary = false, + .enable_int96_timestamps = false}; + VParquetTransformer transformer(&state, file_writer.get(), output_exprs, {"shape"}, false, + options, &schema_json, schema.get()); + ASSERT_TRUE(transformer.open().ok()); + + auto geometry_column = ColumnSpatial::create(TYPE_GEOMETRY); + geometry_column->insert_data("\x01", 1); + Block block; + block.insert(ColumnWithTypeAndName(std::move(geometry_column), geometry_type, "shape")); + EXPECT_FALSE(transformer.write(block).ok()); +} + +TEST_F(VParquetTransformerTest, RejectsNestedIcebergSpatialColumns) { + auto geometry_type = std::make_shared(TYPE_GEOMETRY, "EPSG:3857"); + auto array_type = std::make_shared(geometry_type); + VExprContextSPtrs output_exprs = MockSlotRef::create_mock_contexts(DataTypes {array_type}); + + const std::string schema_json = R"JSON({ + "type": "struct", + "fields": [ + {"id": 1, "name": "shapes", "required": false, + "type": {"type": "list", "element-id": 2, "element": "geometry(EPSG:3857)", + "element-required": false}} + ] + })JSON"; + const auto schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("UTC"); + ParquetFileOptions options {.compression_type = TParquetCompressionType::UNCOMPRESSED, + .parquet_version = TParquetVersion::PARQUET_1_0, + .parquet_disable_dictionary = false, + .enable_int96_timestamps = false}; + VParquetTransformer transformer(&state, file_writer.get(), output_exprs, {"shapes"}, false, + options, &schema_json, schema.get()); + const Status status = transformer.open(); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string(status.msg()).find("Nested Iceberg spatial columns"), std::string::npos); +} + TEST_F(VParquetTransformerTest, WritesNestedIcebergVariant) { auto variant_type = std::make_shared(); auto array_type = std::make_shared(variant_type); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 4ae98f0d4cdbbd..1ca02551a77e29 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -28,6 +28,7 @@ #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_spatial.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_variant_v2.h" #include "core/data_type/primitive_type.h" @@ -941,6 +942,43 @@ TEST(ParquetSchemaTest, NativeStringAnnotationsAndTimeUnitsPreserveLogicalTypes) EXPECT_EQ(remove_nullable(descriptor.get_column(7)->data_type)->get_scale(), 6); } +TEST(ParquetSchemaTest, NativeSpatialAnnotationsPreserveMetadata) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + + auto binary_leaf = [](const std::string& name) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + leaf.__set_logicalType(tparquet::LogicalType()); + return leaf; + }; + auto geometry = binary_leaf("shape"); + geometry.logicalType.__set_GEOMETRY(tparquet::GeometryType()); + geometry.logicalType.GEOMETRY.__set_crs("EPSG:3857"); + + auto geography = binary_leaf("place"); + geography.logicalType.__set_GEOGRAPHY(tparquet::GeographyType()); + geography.logicalType.GEOGRAPHY.__set_crs("OGC:CRS84"); + geography.logicalType.GEOGRAPHY.__set_algorithm(tparquet::EdgeInterpolationAlgorithm::VINCENTY); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, geometry, geography}).ok()); + + const auto& geometry_type = assert_cast( + *remove_nullable(descriptor.get_column(0)->data_type)); + EXPECT_EQ(TYPE_GEOMETRY, geometry_type.get_primitive_type()); + EXPECT_EQ("EPSG:3857", geometry_type.crs()); + + const auto& geography_type = assert_cast( + *remove_nullable(descriptor.get_column(1)->data_type)); + EXPECT_EQ(TYPE_GEOGRAPHY, geography_type.get_primitive_type()); + EXPECT_EQ("OGC:CRS84", geography_type.crs()); + EXPECT_EQ("vincenty", geography_type.algorithm()); +} + TEST(ParquetSchemaTest, NativeSchemaRejectsAmbiguousKindsAndMissingRepetition) { auto valid_root = []() { tparquet::SchemaElement root; diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 216db92e21698a..c1707991633042 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -81,7 +81,9 @@ public enum PrimitiveType { VARIANT("VARIANT", 24, TPrimitiveType.VARIANT, false), TEMPLATE("TEMPLATE", -1, TPrimitiveType.INVALID_TYPE, false), // Unsupported scalar types. - BINARY("BINARY", -1, TPrimitiveType.BINARY, false); + BINARY("BINARY", -1, TPrimitiveType.BINARY, false), + GEOMETRY("GEOMETRY", 16, TPrimitiveType.GEOMETRY, false), + GEOGRAPHY("GEOGRAPHY", 16, TPrimitiveType.GEOGRAPHY, false); private static final int DATE_INDEX_LEN = 3; @@ -259,6 +261,10 @@ public static PrimitiveType fromThrift(TPrimitiveType tPrimitiveType) { return VARIANT; case VARBINARY: return VARBINARY; + case GEOMETRY: + return GEOMETRY; + case GEOGRAPHY: + return GEOGRAPHY; case ALL: default: return INVALID_TYPE; diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java index 219a7c2384fc6c..c51b60af945ef8 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -115,6 +115,10 @@ public long getByteSize() { @SerializedName(value = "scale") private int scale; + @SerializedName(value = "spatialCrs") + private String spatialCrs; + @SerializedName(value = "spatialAlgorithm") + private String spatialAlgorithm; // Only used for alias function decimal @SerializedName(value = "precisionStr") private String precisionStr; @@ -152,6 +156,10 @@ public static ScalarType createType(PrimitiveType type, int len, int precision, return createTimeStampTzType(scale); case VARBINARY: return createVarbinaryType(len); + case GEOMETRY: + return createGeometryType(); + case GEOGRAPHY: + return createGeographyType(); default: return createType(type); } @@ -227,6 +235,10 @@ public static ScalarType createType(PrimitiveType type) { return IPV6; case VARBINARY: return VARBINARY; + case GEOMETRY: + return createGeometryType(); + case GEOGRAPHY: + return createGeographyType(); default: LOG.warn("type={}", type); Preconditions.checkState(false, "type.name()=" + type.name()); @@ -532,6 +544,43 @@ public static Type getDefaultDateType(Type type) { } } + public static ScalarType createGeometryType() { + return createGeometryType("OGC:CRS84"); + } + + public static ScalarType createGeometryType(String crs) { + Preconditions.checkArgument(crs != null && !crs.trim().isEmpty(), "Spatial CRS must not be empty"); + ScalarType result = new ScalarType(PrimitiveType.GEOMETRY); + result.spatialCrs = crs; + return result; + } + + public static ScalarType createGeographyType() { + return createGeographyType("OGC:CRS84", "spherical"); + } + + public static ScalarType createGeographyType(String crs, String algorithm) { + Preconditions.checkArgument(crs != null && !crs.trim().isEmpty(), "Spatial CRS must not be empty"); + Preconditions.checkArgument(algorithm != null && !algorithm.trim().isEmpty(), + "Geography algorithm must not be empty"); + ScalarType result = new ScalarType(PrimitiveType.GEOGRAPHY); + result.spatialCrs = crs; + result.spatialAlgorithm = algorithm; + return result; + } + + public String getSpatialCrs() { + return spatialCrs; + } + + public String getSpatialAlgorithm() { + return spatialAlgorithm; + } + + public boolean isSpatialType() { + return type == PrimitiveType.GEOMETRY || type == PrimitiveType.GEOGRAPHY; + } + public static ScalarType createVarbinaryType(int len) { // length checked in analysis ScalarType type = new ScalarType(PrimitiveType.VARBINARY); @@ -595,6 +644,9 @@ public static ScalarType createHllType() { @Override public String toString() { + if (isSpatialType()) { + return toSql(0); + } if (type == PrimitiveType.CHAR) { if (isWildcardChar()) { return "character(" + MAX_CHAR_LENGTH + ")"; @@ -631,6 +683,11 @@ public String toString() { @Override public String toSql(int depth) { + if (isSpatialType()) { + return type.name() + "('" + spatialCrs.replace("'", "''") + "'" + + (type == PrimitiveType.GEOGRAPHY ? ", '" + spatialAlgorithm.replace("'", "''") + "'" : "") + + ")"; + } StringBuilder stringBuilder = new StringBuilder(); switch (type) { case CHAR: @@ -749,6 +806,12 @@ public void toThrift(TTypeDesc container) { node.setType(TTypeNodeType.SCALAR); TScalarType scalarType = new TScalarType(); scalarType.setType(type.toThrift()); + if (isSpatialType()) { + scalarType.setSpatialCrs(spatialCrs); + if (type == PrimitiveType.GEOGRAPHY) { + scalarType.setSpatialAlgorithm(spatialAlgorithm); + } + } container.setByteSize(byteSize); switch (type) { @@ -966,6 +1029,10 @@ public boolean equals(Object o) { if (type != other.type) { return false; } + if (isSpatialType()) { + return Objects.equals(spatialCrs, other.spatialCrs) + && Objects.equals(spatialAlgorithm, other.spatialAlgorithm); + } if (type == PrimitiveType.CHAR) { return len == other.len; } @@ -998,6 +1065,9 @@ public TColumnType toColumnTypeThrift() { @Override public int hashCode() { + if (isSpatialType()) { + return Objects.hash(type, spatialCrs, spatialAlgorithm); + } int result = 0; result = 31 * result + Objects.hashCode(type); result = 31 * result + precision; diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index fc214943541985..c291a3a1aea973 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -965,7 +965,16 @@ protected static Pair fromThrift(TTypeDesc col, int nodeIdx) { case SCALAR: { Preconditions.checkState(node.isSetScalarType()); TScalarType scalarType = node.getScalarType(); - if (scalarType.getType() == TPrimitiveType.CHAR) { + if (scalarType.getType() == TPrimitiveType.GEOMETRY) { + Preconditions.checkArgument(scalarType.isSetSpatialCrs(), "Missing geometry CRS"); + Preconditions.checkArgument(!scalarType.isSetSpatialAlgorithm(), + "Geometry must not specify a geography algorithm"); + type = ScalarType.createGeometryType(scalarType.getSpatialCrs()); + } else if (scalarType.getType() == TPrimitiveType.GEOGRAPHY) { + Preconditions.checkArgument(scalarType.isSetSpatialCrs() && scalarType.isSetSpatialAlgorithm(), + "Missing geography CRS or algorithm"); + type = ScalarType.createGeographyType(scalarType.getSpatialCrs(), scalarType.getSpatialAlgorithm()); + } else if (scalarType.getType() == TPrimitiveType.CHAR) { Preconditions.checkState(scalarType.isSetLen()); type = ScalarType.createCharType(scalarType.getLen()); } else if (scalarType.getType() == TPrimitiveType.VARCHAR) { diff --git a/fe/fe-common/src/test/java/org/apache/doris/catalog/SpatialTypeTest.java b/fe/fe-common/src/test/java/org/apache/doris/catalog/SpatialTypeTest.java new file mode 100644 index 00000000000000..9fe60255787778 --- /dev/null +++ b/fe/fe-common/src/test/java/org/apache/doris/catalog/SpatialTypeTest.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.thrift.TTypeDesc; + +import org.junit.Assert; +import org.junit.Test; + +public class SpatialTypeTest { + @Test + public void testRoundTrip() { + for (ScalarType type : new ScalarType[] { + ScalarType.createGeometryType(), ScalarType.createGeometryType("EPSG:3857"), + ScalarType.createGeographyType(), ScalarType.createGeographyType("OGC:CRS84", "ellipsoidal")}) { + Type restored = Type.fromThrift(type.toThrift()); + Assert.assertEquals(type, restored); + Assert.assertEquals(type.hashCode(), restored.hashCode()); + Assert.assertEquals(type.toSql(), restored.toSql()); + Assert.assertFalse(type.getPrimitiveType().isAvailableInDdl()); + } + } + + @Test + public void testParametersAffectIdentity() { + Assert.assertNotEquals(ScalarType.createGeometryType(), ScalarType.createGeometryType("EPSG:3857")); + Assert.assertNotEquals(ScalarType.createGeometryType(), ScalarType.createGeographyType()); + Assert.assertNotEquals(ScalarType.createGeographyType(), + ScalarType.createGeographyType("OGC:CRS84", "ellipsoidal")); + Assert.assertEquals("GEOMETRY('OGC:CRS84')", ScalarType.createGeometryType().toSql()); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectMissingCrs() { + TTypeDesc thrift = ScalarType.createGeometryType().toThrift(); + thrift.getTypes().get(0).getScalarType().unsetSpatialCrs(); + Type.fromThrift(thrift); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectEmptyAlgorithm() { + ScalarType.createGeographyType("OGC:CRS84", ""); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 21ec0d1179d445..e5eaa3cb078460 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -421,9 +421,11 @@ public void analyze(boolean isOlap) throws AnalysisException { FeNameFormat.checkColumnName(name); FeNameFormat.checkColumnCommentLength(comment); - typeDef.analyze(); - Type type = typeDef.getType(); + if (isOlap && containsSpatialType(type)) { + throw new AnalysisException("GEOMETRY and GEOGRAPHY are not supported for Doris internal tables"); + } + typeDef.analyze(); if (!Config.enable_quantile_state_type && type.isQuantileStateType()) { throw new AnalysisException("quantile_state is disabled" @@ -556,6 +558,24 @@ public void analyze(boolean isOlap) throws AnalysisException { validateGeneratedColumnInfo(); } + private static boolean containsSpatialType(Type type) { + if (type instanceof ScalarType) { + return ((ScalarType) type).isSpatialType(); + } + if (type instanceof org.apache.doris.catalog.ArrayType) { + return containsSpatialType(((org.apache.doris.catalog.ArrayType) type).getItemType()); + } + if (type instanceof org.apache.doris.catalog.MapType) { + org.apache.doris.catalog.MapType mapType = (org.apache.doris.catalog.MapType) type; + return containsSpatialType(mapType.getKeyType()) || containsSpatialType(mapType.getValueType()); + } + if (type instanceof org.apache.doris.catalog.StructType) { + return ((org.apache.doris.catalog.StructType) type).getFields().stream() + .anyMatch(field -> containsSpatialType(field.getType())); + } + return false; + } + @SuppressWarnings("checkstyle:Indentation") public static void validateDefaultValue(Type type, String defaultValue, DefaultValueExprDef defaultValueExprDef) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index a27886f909afca..f2b0e34f7b1011 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -481,8 +481,10 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.StDisjoint; import org.apache.doris.nereids.trees.expressions.functions.scalar.StDistance; import org.apache.doris.nereids.trees.expressions.functions.scalar.StDistanceSphere; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeogFromWKB; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeomFromWKB; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryFromWKB; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryFromWKBTyped; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryType; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryfromtext; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeomfromtext; @@ -1074,6 +1076,8 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(StGeometryFromWKB.class, "st_geometryfromwkb"), scalar(StGeomfromtext.class, "st_geomfromtext"), scalar(StGeomFromWKB.class, "st_geomfromwkb"), + scalar(StGeogFromWKB.class, "st_geogfromwkb"), + scalar(StGeometryFromWKBTyped.class, "st_geometryfromwkbtyped"), scalar(StLinefromtext.class, "st_linefromtext"), scalar(StLinestringfromtext.class, "st_linestringfromtext"), scalar(StPoint.class, "st_point"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java index eba180512b475e..b5614d73a7099b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java @@ -26,6 +26,7 @@ import org.apache.doris.datasource.DorisTypeVisitor; import com.google.common.collect.Lists; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -136,6 +137,16 @@ public Type atomic(org.apache.doris.catalog.Type atomic) { return Types.TimestampType.withZone(); } else if (primitiveType.equals(PrimitiveType.VARIANT)) { return Types.VariantType.get(); + } else if (primitiveType.equals(PrimitiveType.GEOMETRY)) { + return Types.GeometryType.of(((ScalarType) atomic).getSpatialCrs()); + } else if (primitiveType.equals(PrimitiveType.GEOGRAPHY)) { + ScalarType geography = (ScalarType) atomic; + if (Types.GeographyType.DEFAULT_CRS.equalsIgnoreCase(geography.getSpatialCrs()) + && EdgeAlgorithm.SPHERICAL.toString().equalsIgnoreCase(geography.getSpatialAlgorithm())) { + return Types.GeographyType.crs84(); + } + return Types.GeographyType.of(geography.getSpatialCrs(), + EdgeAlgorithm.fromName(geography.getSpatialAlgorithm())); } // unsupported type: PrimitiveType.HLL BITMAP BINARY diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 74a3b366f87721..b458ceb8eb15fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.ColumnType; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MapType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; import org.apache.doris.common.DdlException; @@ -382,6 +383,11 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx && IcebergUtils.containsVariant(column.getType()))) { throw new UserException("Iceberg VARIANT DDL currently supports only top-level columns"); } + if (columns.stream().anyMatch(column -> (!(column.getType() instanceof ScalarType) + || !((ScalarType) column.getType()).isSpatialType()) + && IcebergUtils.containsSpatial(column.getType()))) { + throw new UserException("Iceberg GEOMETRY and GEOGRAPHY DDL currently support only top-level columns"); + } List collect = columns.stream() .map(col -> new StructField(col.getName(), col.getType(), col.getComment(), col.isAllowNull())) .collect(Collectors.toList()); @@ -396,7 +402,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx && !IcebergUtils.hasIcebergCatalogFormatVersion(catalogProperties)) { properties.put(TableProperties.FORMAT_VERSION, "2"); } - if (columns.stream().anyMatch(column -> IcebergUtils.containsVariant(column.getType()))) { + if (columns.stream().anyMatch(column -> IcebergUtils.containsVariant(column.getType()) + || IcebergUtils.containsSpatial(column.getType()))) { IcebergUtils.validateWriteSchema(columns, IcebergUtils.getEffectiveIcebergFormatVersion(properties, catalogProperties), IcebergUtils.getEffectiveFileFormat(properties, catalogProperties)); @@ -711,6 +718,25 @@ private void validateVariantSchema(Table icebergTable, org.apache.doris.catalog. } } + private void validateSpatialSchema(Table icebergTable, org.apache.doris.catalog.Type dorisType, + String columnPath, boolean nestedColumn) throws UserException { + if (!IcebergUtils.containsSpatial(dorisType)) { + return; + } + if (nestedColumn || !(dorisType instanceof ScalarType) + || !((ScalarType) dorisType).isSpatialType()) { + throw new UserException("Iceberg GEOMETRY and GEOGRAPHY DDL currently support only top-level columns"); + } + if (IcebergUtils.getFormatVersion(icebergTable) < IcebergUtils.ICEBERG_SPATIAL_MIN_VERSION) { + throw new UserException("Iceberg spatial column " + columnPath + + " requires table format-version 3"); + } + if (IcebergUtils.getFileFormat(icebergTable) != org.apache.iceberg.FileFormat.PARQUET) { + throw new UserException("Iceberg spatial column " + columnPath + + " requires Parquet data files"); + } + } + private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, ColumnPath columnPath, Schema schema, String operation) throws UserException { String columnName = columnPath.getFullPath(); @@ -770,6 +796,7 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po validateAddColumnMetadata(column, true); Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); + validateSpatialSchema(icebergTable, column.getType(), column.getName(), false); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); validateNoCaseInsensitiveSiblingCollision( @@ -801,6 +828,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co } Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); + validateSpatialSchema(icebergTable, column.getType(), columnPath.getFullPath(), true); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); if (!parentPath.getType().isStructType()) { throw new UserException("Parent column path '" + columnPath.getParentPathString() @@ -833,6 +861,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda for (Column column : columns) { validateAddColumnMetadata(column, true); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); + validateSpatialSchema(icebergTable, column.getType(), column.getName(), false); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); } validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); @@ -1008,6 +1037,7 @@ private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPat // contain VARIANT columns even though Doris cannot write VARIANT values to ORC files. validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), false, !variantModify); + validateSpatialSchema(icebergTable, column.getType(), columnPath.getFullPath(), false); org.apache.iceberg.types.Type targetType; if (variantModify) { validateForModifyVariantColumn(column, currentCol, resolvedPath.getFullPath()); @@ -1071,6 +1101,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column validateNestedModifyColumnMetadata(column, resolvedPath.getFullPath()); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); + validateSpatialSchema(icebergTable, column.getType(), columnPath.getFullPath(), true); org.apache.iceberg.types.Type targetType; if (column.getType().isComplexType()) { validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSpatialWriteAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSpatialWriteAnalyzer.java new file mode 100644 index 00000000000000..3fd51603638c97 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSpatialWriteAnalyzer.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.types.DataType; + +import java.util.List; +import java.util.Objects; + +/** Analysis checks that preserve Iceberg spatial type parameters during writes. */ +public final class IcebergSpatialWriteAnalyzer { + private IcebergSpatialWriteAnalyzer() { + } + + /** + * Rejects an INSERT source whose spatial kind or parameters differ from the Iceberg target + * before sink coercion can erase that distinction. + */ + public static void validate( + List targetColumns, List sourceColumns) { + if (targetColumns.size() != sourceColumns.size()) { + throw new AnalysisException("Iceberg spatial write target and source columns are not aligned"); + } + for (int i = 0; i < targetColumns.size(); ++i) { + Type targetCatalogType = targetColumns.get(i).getType(); + if (!(targetCatalogType instanceof ScalarType) + || !((ScalarType) targetCatalogType).isSpatialType()) { + continue; + } + validateSpatialConversion(sourceColumns.get(i).getDataType().toCatalogDataType(), + (ScalarType) targetCatalogType, targetColumns.get(i).getName()); + } + } + + /** Validates each MERGE action before a target cast can hide its spatial source type. */ + public static void validateMergeActions( + List targetColumns, List sourceColumns) { + if (targetColumns.size() != sourceColumns.size()) { + throw new AnalysisException("Iceberg spatial write target and source columns are not aligned"); + } + for (int i = 0; i < targetColumns.size(); ++i) { + Type targetCatalogType = targetColumns.get(i).getType(); + if (targetCatalogType instanceof ScalarType && ((ScalarType) targetCatalogType).isSpatialType()) { + validateMergeActionExpression(sourceColumns.get(i), (ScalarType) targetCatalogType, + targetColumns.get(i).getName()); + } + } + } + + private static void validateMergeActionExpression( + Expression expression, ScalarType targetType, String columnName) { + if (expression instanceof Alias) { + validateMergeActionExpression(expression.child(0), targetType, columnName); + return; + } + DataType targetDataType = DataType.fromCatalogType(targetType); + if (expression instanceof If && expression.getDataType().equals(targetDataType)) { + If ifExpression = (If) expression; + validateMergeActionExpression(ifExpression.getTrueValue(), targetType, columnName); + validateMergeActionExpression(ifExpression.getFalseValue(), targetType, columnName); + return; + } + if (expression instanceof Cast && expression.getDataType().equals(targetDataType)) { + validateSpatialConversion(expression.child(0).getDataType().toCatalogDataType(), targetType, columnName); + return; + } + validateSpatialConversion(expression.getDataType().toCatalogDataType(), targetType, columnName); + } + + static void validateSpatialConversion(Type sourceType, ScalarType targetType, String columnName) { + if (sourceType.isNull()) { + return; + } + if (!(sourceType instanceof ScalarType) || !((ScalarType) sourceType).isSpatialType()) { + throw new AnalysisException("Iceberg spatial write cannot convert input column '" + columnName + + "' from " + sourceType.toSql() + " to " + targetType.toSql()); + } + ScalarType sourceSpatialType = (ScalarType) sourceType; + if (sourceSpatialType.getPrimitiveType() != targetType.getPrimitiveType()) { + throw new AnalysisException("Iceberg spatial write type mismatch for column '" + columnName + + "': source " + sourceSpatialType.toSql() + " does not match target " + targetType.toSql()); + } + if (!Objects.equals(sourceSpatialType.getSpatialCrs(), targetType.getSpatialCrs())) { + throw new AnalysisException("Iceberg spatial write CRS mismatch for column '" + columnName + + "': source " + sourceSpatialType.getSpatialCrs() + " does not match target " + + targetType.getSpatialCrs()); + } + if (!Objects.equals(sourceSpatialType.getSpatialAlgorithm(), targetType.getSpatialAlgorithm())) { + throw new AnalysisException("Iceberg spatial write algorithm mismatch for column '" + columnName + + "': source " + sourceSpatialType.getSpatialAlgorithm() + " does not match target " + + targetType.getSpatialAlgorithm()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index da4513ff8a1d7e..dc15d5d72ec4fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -121,6 +121,7 @@ import org.apache.iceberg.mapping.NameMapping; import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -255,6 +256,7 @@ private static StorageProperties chooseS3CompatibleStorage(List containsSpatial(field.getType())); + } + return false; + } + + private static boolean containsNestedSpatial(Type type) { + if (type.isScalarType()) { + return false; + } + return containsSpatial(type); + } + public static void validateWriteSchema(Table table, List columns) { boolean writesVariant = columns.stream().anyMatch(column -> containsVariant(column.getType())); + boolean writesSpatial = columns.stream().anyMatch(column -> containsSpatial(column.getType())); FileFormat fileFormat = getFileFormat(table); - if (writesVariant) { + if (writesVariant || writesSpatial) { validateWriteSchema(columns, getFormatVersion(table), fileFormat); + } + if (writesVariant) { validateVariantWriteProperties(columns, table.properties()); } boolean writesOrcBinary = fileFormat == FileFormat.ORC @@ -832,16 +876,36 @@ static void validateVariantWriteProperties(List columns, Map columns, int formatVersion, FileFormat fileFormat) { - if (columns.stream().noneMatch(column -> containsVariant(column.getType()))) { + boolean hasVariant = columns.stream().anyMatch(column -> containsVariant(column.getType())); + boolean hasSpatial = columns.stream().anyMatch(column -> containsSpatial(column.getType())); + if (!hasVariant && !hasSpatial) { return; } - if (formatVersion < ICEBERG_VARIANT_MIN_VERSION) { + if (columns.stream().anyMatch(column -> containsNestedSpatial(column.getType()))) { throw new org.apache.doris.nereids.exceptions.AnalysisException( - "Iceberg VARIANT writes require table format-version 3, but found " + formatVersion); + "Iceberg writes do not support GEOMETRY or GEOGRAPHY nested in complex types"); } - if (fileFormat != FileFormat.PARQUET) { - throw new org.apache.doris.nereids.exceptions.AnalysisException( - "Iceberg VARIANT writes require Parquet data files, but found " + fileFormat); + if (hasVariant) { + if (formatVersion < ICEBERG_VARIANT_MIN_VERSION) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Iceberg VARIANT writes require table format-version 3, but found " + formatVersion); + } + if (fileFormat != FileFormat.PARQUET) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Iceberg VARIANT writes require Parquet data files, but found " + fileFormat); + } + } + if (hasSpatial) { + if (formatVersion < ICEBERG_SPATIAL_MIN_VERSION) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Iceberg GEOMETRY and GEOGRAPHY writes require table format-version 3, but found " + + formatVersion); + } + if (fileFormat != FileFormat.PARQUET) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Iceberg GEOMETRY and GEOGRAPHY writes require Parquet data files, but found " + + fileFormat); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index d145e09c875e4e..f727c6d3b523aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -23,6 +23,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.datasource.VariantWritePlanValidator; import org.apache.doris.datasource.iceberg.IcebergMergeOperation; +import org.apache.doris.datasource.iceberg.IcebergSpatialWriteAnalyzer; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.IcebergVariantWriteAnalyzer; import org.apache.doris.nereids.CTEContext; @@ -364,6 +365,7 @@ static List coerceIcebergMergeOutput(List sinkColumns, } if (writesDataFiles) { IcebergVariantWriteAnalyzer.validateMergeActions(visibleColumns, visibleOutputExprs); + IcebergSpatialWriteAnalyzer.validateMergeActions(visibleColumns, visibleOutputExprs); } List castExprs = Lists.newArrayListWithCapacity(outputExprs.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 2479c271b1b372..625dd5c50d3184 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -39,6 +39,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergSpatialWriteAnalyzer; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.IcebergVariantWriteAnalyzer; import org.apache.doris.datasource.iceberg.IcebergWriteSchemaContext; @@ -857,6 +858,7 @@ private Plan bindIcebergTableSink(MatchingContext> } IcebergVariantWriteAnalyzer.validate(bindColumns, child.getOutput()); + IcebergSpatialWriteAnalyzer.validate(bindColumns, child.getOutput()); VariantWritePlanValidator.validateNoLossyCoercion( "Iceberg", bindColumns, child, ctx.cascadesContext.getCteContext()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SpatialFunctionSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SpatialFunctionSignature.java new file mode 100644 index 00000000000000..bc1e815edb788c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SpatialFunctionSignature.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.GeographyType; +import org.apache.doris.nereids.types.GeometryType; + +/** Helpers for preserving parameterized spatial types in function signatures. */ +final class SpatialFunctionSignature { + private SpatialFunctionSignature() { + } + + static FunctionSignature unary(Expression argument, DataType returnType) { + if (!isSpatial(argument)) { + return null; + } + return FunctionSignature.ret(returnType).args(argument.getDataType()); + } + + static FunctionSignature binary(Expression left, Expression right, DataType returnType) { + if (!isSpatial(left) || !isSpatial(right)) { + return null; + } + return FunctionSignature.ret(returnType).args(left.getDataType(), right.getDataType()); + } + + private static boolean isSpatial(Expression expression) { + return expression.getDataType() instanceof GeometryType + || expression.getDataType() instanceof GeographyType; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAsBinary.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAsBinary.java index 58c7bb0f535017..8459be45197e3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAsBinary.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAsBinary.java @@ -68,6 +68,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.unary(child(0), VarcharType.SYSTEM_DEFAULT); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStAsBinary(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java index 2e4d2b35dbb50a..2498c662dfeb29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAstext.java @@ -24,6 +24,8 @@ import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.GeographyType; +import org.apache.doris.nereids.types.GeometryType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.VarcharType; @@ -69,6 +71,15 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + if (child(0).getDataType() instanceof GeometryType + || child(0).getDataType() instanceof GeographyType) { + return FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(child(0).getDataType()); + } + return ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStAstext(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAswkt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAswkt.java index 6101798f47b038..2d55df9889712e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAswkt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StAswkt.java @@ -68,6 +68,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.unary(child(0), VarcharType.SYSTEM_DEFAULT); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStAswkt(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java index 1f37930d5655f2..8ef4138c6a7aee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StContains.java @@ -68,6 +68,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.binary(child(0), child(1), BooleanType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStContains(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDisjoint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDisjoint.java index d0ea87ab93161c..b78f97090c6b87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDisjoint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDisjoint.java @@ -70,6 +70,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.binary(child(0), child(1), BooleanType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStDisjoint(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistance.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistance.java index dbed7879d6b069..1d58d439ceeebc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistance.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StDistance.java @@ -68,6 +68,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.binary(child(0), child(1), DoubleType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStDistance(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeogFromWKB.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeogFromWKB.java new file mode 100644 index 00000000000000..719d63c31bbe0d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeogFromWKB.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.GeographyType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarcharType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** ScalarFunction 'st_geogfromwkb'. */ +public class StGeogFromWKB extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(new GeographyType("OGC:CRS84", "spherical")) + .args(VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(new GeographyType("OGC:CRS84", "spherical")) + .args(StringType.INSTANCE) + ); + + public StGeogFromWKB(Expression arg) { + super("st_geogfromwkb", arg); + } + + private StGeogFromWKB(ScalarFunctionParams functionParams) { + super(functionParams); + } + + @Override + public StGeogFromWKB withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new StGeogFromWKB(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitStGeogfromwkb(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryFromWKBTyped.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryFromWKBTyped.java new file mode 100644 index 00000000000000..98eb80b439f1ca --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryFromWKBTyped.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.GeometryType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarcharType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** ScalarFunction 'st_geometryfromwkbtyped'. */ +public class StGeometryFromWKBTyped extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(new GeometryType("OGC:CRS84")).args(VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(new GeometryType("OGC:CRS84")).args(StringType.INSTANCE) + ); + + public StGeometryFromWKBTyped(Expression arg) { + super("st_geometryfromwkbtyped", arg); + } + + private StGeometryFromWKBTyped(ScalarFunctionParams functionParams) { + super(functionParams); + } + + @Override + public StGeometryFromWKBTyped withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new StGeometryFromWKBTyped(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitStGeometryfromwkbtyped(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryType.java index 882c07182328ec..2abcb96a976caf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StGeometryType.java @@ -67,6 +67,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.unary(child(0), VarcharType.SYSTEM_DEFAULT); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStGeometryType(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StIntersects.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StIntersects.java index c2380d9e1d7ba1..11daf31d953e7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StIntersects.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StIntersects.java @@ -70,6 +70,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.binary(child(0), child(1), BooleanType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStIntersects(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLength.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLength.java index 963de26cea7b61..23b273884c33af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLength.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StLength.java @@ -68,6 +68,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.unary(child(0), DoubleType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStLength(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StTouches.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StTouches.java index c7a6fa8d070bfa..6ebb8c510acb5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StTouches.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StTouches.java @@ -70,6 +70,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.binary(child(0), child(1), BooleanType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStTouches(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java index ce803da1f32759..1dc837fbe763cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StX.java @@ -70,6 +70,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.unary(child(0), DoubleType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStX(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java index e6369ab1608fb9..d530fd8828fc81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StY.java @@ -70,6 +70,12 @@ public List getSignatures() { return SIGNATURES; } + @Override + public FunctionSignature searchSignature(List signatures) { + FunctionSignature signature = SpatialFunctionSignature.unary(child(0), DoubleType.INSTANCE); + return signature != null ? signature : ExplicitlyCastableSignature.super.searchSignature(signatures); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStY(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index e071bed608b606..e35a610be21f49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -500,8 +500,10 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.StDisjoint; import org.apache.doris.nereids.trees.expressions.functions.scalar.StDistance; import org.apache.doris.nereids.trees.expressions.functions.scalar.StDistanceSphere; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeogFromWKB; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeomFromWKB; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryFromWKB; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryFromWKBTyped; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryType; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeometryfromtext; import org.apache.doris.nereids.trees.expressions.functions.scalar.StGeomfromtext; @@ -2463,6 +2465,15 @@ default R visitStGeomfromwkb(StGeomFromWKB stGeomfromwkb, C context) { return visitScalarFunction(stGeomfromwkb, context); } + default R visitStGeogfromwkb(StGeogFromWKB stGeogfromwkb, C context) { + return visitScalarFunction(stGeogfromwkb, context); + } + + default R visitStGeometryfromwkbtyped(StGeometryFromWKBTyped stGeometryfromwkbtyped, + C context) { + return visitScalarFunction(stGeometryfromwkbtyped, context); + } + default R visitStAsBinary(StAsBinary stAsBinary, C context) { return visitScalarFunction(stAsBinary, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 81374de189f341..af69970346331f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -380,6 +381,9 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); } + if (isOlap && containsSpatialType(type)) { + throw new AnalysisException("GEOMETRY and GEOGRAPHY are not supported for Doris internal tables"); + } type.validateDataType(); type = updateCharacterTypeLength(type); if (type.isArrayType()) { @@ -572,6 +576,25 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c validateGeneratedColumnInfo(); } + private static boolean containsSpatialType(DataType type) { + PrimitiveType primitiveType = type.toCatalogDataType().getPrimitiveType(); + if (primitiveType == PrimitiveType.GEOMETRY || primitiveType == PrimitiveType.GEOGRAPHY) { + return true; + } + if (type.isArrayType()) { + return containsSpatialType(((ArrayType) type).getItemType()); + } + if (type.isMapType()) { + MapType mapType = (MapType) type; + return containsSpatialType(mapType.getKeyType()) || containsSpatialType(mapType.getValueType()); + } + if (type.isStructType()) { + return ((StructType) type).getFields().stream() + .anyMatch(field -> containsSpatialType(field.getDataType())); + } + return false; + } + /** * Validate non-null defaults for complex types before connector-specific validation. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index 35fa06b5d17139..85c972b249d9c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -414,6 +414,11 @@ public static DataType convertFromString(String type) { @Developing // should support map, struct public static DataType fromCatalogType(Type type) { switch (type.getPrimitiveType()) { + case GEOMETRY: + return new GeometryType(((ScalarType) type).getSpatialCrs()); + case GEOGRAPHY: + return new GeographyType(((ScalarType) type).getSpatialCrs(), + ((ScalarType) type).getSpatialAlgorithm()); case BOOLEAN: return BooleanType.INSTANCE; case TINYINT: return TinyIntType.INSTANCE; case SMALLINT: return SmallIntType.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/GeographyType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/GeographyType.java new file mode 100644 index 00000000000000..c88989a8c4f93c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/GeographyType.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.types.coercion.PrimitiveType; + +import java.util.Objects; + +/** Parameterized Iceberg geography type. */ +public final class GeographyType extends PrimitiveType { + private final String crs; + private final String algorithm; + + public GeographyType(String crs, String algorithm) { + ScalarType validated = ScalarType.createGeographyType(crs, algorithm); + this.crs = validated.getSpatialCrs(); + this.algorithm = validated.getSpatialAlgorithm(); + } + + @Override + public Type toCatalogDataType() { + return ScalarType.createGeographyType(crs, algorithm); + } + + @Override + public String toSql() { + return toCatalogDataType().toSql(); + } + + @Override + public int width() { + return 16; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof GeographyType)) { + return false; + } + GeographyType that = (GeographyType) other; + return crs.equals(that.crs) && algorithm.equals(that.algorithm); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), crs, algorithm); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/GeometryType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/GeometryType.java new file mode 100644 index 00000000000000..cb7135c5f1fbb6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/GeometryType.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.types.coercion.PrimitiveType; + +import java.util.Objects; + +/** Parameterized Iceberg geometry type. */ +public final class GeometryType extends PrimitiveType { + private final String crs; + + public GeometryType(String crs) { + ScalarType validated = ScalarType.createGeometryType(crs); + this.crs = validated.getSpatialCrs(); + } + + @Override + public Type toCatalogDataType() { + return ScalarType.createGeometryType(crs); + } + + @Override + public String toSql() { + return toCatalogDataType().toSql(); + } + + @Override + public int width() { + return 16; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof GeometryType)) { + return false; + } + GeometryType that = (GeometryType) other; + return crs.equals(that.crs); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), crs); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSpatialWriteAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSpatialWriteAnalyzerTest.java new file mode 100644 index 00000000000000..0a6b6155a634b6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSpatialWriteAnalyzerTest.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.types.DataType; + +import com.google.common.collect.ImmutableList; +import org.junit.Assert; +import org.junit.Test; + +public class IcebergSpatialWriteAnalyzerTest { + @Test + public void testMatchingSpatialParametersAreAccepted() { + validate(ScalarType.createGeometryType("EPSG:4326"), + ScalarType.createGeometryType("EPSG:4326")); + validate(ScalarType.createGeographyType("OGC:CRS84", "spherical"), + ScalarType.createGeographyType("OGC:CRS84", "spherical")); + } + + @Test + public void testSpatialKindMismatchIsRejected() { + assertRejected(ScalarType.createGeometryType("OGC:CRS84"), + ScalarType.createGeographyType("OGC:CRS84", "spherical"), "type mismatch"); + } + + @Test + public void testSpatialCrsMismatchIsRejected() { + assertRejected(ScalarType.createGeometryType("EPSG:3857"), + ScalarType.createGeometryType("EPSG:4326"), "CRS mismatch"); + } + + @Test + public void testSpatialAlgorithmMismatchIsRejected() { + assertRejected(ScalarType.createGeographyType("OGC:CRS84", "vincenty"), + ScalarType.createGeographyType("OGC:CRS84", "spherical"), "algorithm mismatch"); + } + + @Test + public void testNonSpatialSourceIsRejected() { + assertRejected(Type.STRING, ScalarType.createGeometryType("OGC:CRS84"), + "cannot convert input column"); + } + + @Test + public void testNullSourceIsAccepted() { + validate(Type.NULL, ScalarType.createGeometryType("OGC:CRS84")); + } + + @Test + public void testMergeValidationInspectsSourceBeforeTargetCast() { + ScalarType targetType = ScalarType.createGeometryType("EPSG:4326"); + DataType targetDataType = DataType.fromCatalogType(targetType); + NamedExpression source = new Alias(new If( + BooleanLiteral.TRUE, + new Cast(new SlotReference("matching", targetDataType), targetDataType), + new Cast(new SlotReference("mismatched", DataType.fromCatalogType( + ScalarType.createGeometryType("EPSG:3857"))), targetDataType)), "payload"); + AnalysisException exception = Assert.assertThrows(AnalysisException.class, + () -> IcebergSpatialWriteAnalyzer.validateMergeActions( + ImmutableList.of(new Column("payload", targetType)), ImmutableList.of(source))); + Assert.assertTrue(exception.getMessage(), exception.getMessage().contains("CRS mismatch")); + Assert.assertTrue(exception.getMessage(), exception.getMessage().contains("payload")); + } + + private static void assertRejected(Type sourceType, ScalarType targetType, String message) { + AnalysisException exception = Assert.assertThrows(AnalysisException.class, + () -> validate(sourceType, targetType)); + Assert.assertTrue(exception.getMessage(), exception.getMessage().contains("payload")); + Assert.assertTrue(exception.getMessage(), exception.getMessage().contains(message)); + } + + private static void validate(Type sourceType, ScalarType targetType) { + NamedExpression source = new Alias( + new SlotReference("source", DataType.fromCatalogType(sourceType)), "payload"); + IcebergSpatialWriteAnalyzer.validate(ImmutableList.of(new Column("payload", targetType)), + ImmutableList.of(source)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 616f1eefa5f3e2..2fd512ec540221 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; @@ -432,6 +433,39 @@ public void testIcebergVariantDefaultsMustBeNull() { Assert.assertTrue(readException.getMessage().contains("VARIANT initial-default must be NULL")); } + @Test + public void testSpatialTypeRoundTrip() { + Type defaultGeometry = IcebergUtils.icebergTypeToDorisType( + Types.GeometryType.crs84(), false, false); + Assert.assertEquals("OGC:CRS84", ((ScalarType) defaultGeometry).getSpatialCrs()); + Assert.assertEquals(Types.GeometryType.crs84(), IcebergUtils.dorisTypeToIcebergType(defaultGeometry)); + + Type geometry = IcebergUtils.icebergTypeToDorisType( + Types.GeometryType.of("EPSG:3857"), false, false); + Assert.assertEquals("EPSG:3857", ((ScalarType) geometry).getSpatialCrs()); + Assert.assertEquals(Types.GeometryType.of("EPSG:3857"), IcebergUtils.dorisTypeToIcebergType(geometry)); + + Type defaultGeography = IcebergUtils.icebergTypeToDorisType( + Types.GeographyType.crs84(), false, false); + Assert.assertEquals("OGC:CRS84", ((ScalarType) defaultGeography).getSpatialCrs()); + Assert.assertEquals("spherical", ((ScalarType) defaultGeography).getSpatialAlgorithm()); + Assert.assertEquals(Types.GeographyType.crs84(), IcebergUtils.dorisTypeToIcebergType(defaultGeography)); + + Type geography = IcebergUtils.icebergTypeToDorisType( + Types.GeographyType.of("EPSG:4326", org.apache.iceberg.types.EdgeAlgorithm.VINCENTY), false, false); + Assert.assertEquals("EPSG:4326", ((ScalarType) geography).getSpatialCrs()); + Assert.assertEquals("vincenty", ((ScalarType) geography).getSpatialAlgorithm()); + Assert.assertEquals(Types.GeographyType.of("EPSG:4326", org.apache.iceberg.types.EdgeAlgorithm.VINCENTY), + IcebergUtils.dorisTypeToIcebergType(geography)); + + Type geographyWithDefaultAlgorithm = IcebergUtils.icebergTypeToDorisType( + Types.GeographyType.of("EPSG:4326"), false, false); + Assert.assertEquals("EPSG:4326", ((ScalarType) geographyWithDefaultAlgorithm).getSpatialCrs()); + Assert.assertEquals("spherical", ((ScalarType) geographyWithDefaultAlgorithm).getSpatialAlgorithm()); + Assert.assertEquals(Types.GeographyType.of("EPSG:4326", org.apache.iceberg.types.EdgeAlgorithm.SPHERICAL), + IcebergUtils.dorisTypeToIcebergType(geographyWithDefaultAlgorithm)); + } + @Test public void testIcebergVariantWriteCapabilityMatrix() { Type variant = IcebergUtils.icebergTypeToDorisType(Types.VariantType.get(), false, false); @@ -456,6 +490,27 @@ public void testIcebergVariantWriteCapabilityMatrix() { ImmutableList.of(nestedColumn), 3, FileFormat.PARQUET); } + @Test + public void testIcebergSpatialWriteCapabilityMatrix() { + Type geometry = IcebergUtils.icebergTypeToDorisType(Types.GeometryType.crs84(), false, false); + Column column = new Column("shape", geometry); + IcebergUtils.validateWriteSchema(ImmutableList.of(column), 3, FileFormat.PARQUET); + + AnalysisException formatException = Assert.assertThrows(AnalysisException.class, + () -> IcebergUtils.validateWriteSchema(ImmutableList.of(column), 2, FileFormat.PARQUET)); + Assert.assertTrue(formatException.getMessage().contains("format-version 3")); + + AnalysisException fileFormatException = Assert.assertThrows(AnalysisException.class, + () -> IcebergUtils.validateWriteSchema(ImmutableList.of(column), 3, FileFormat.ORC)); + Assert.assertTrue(fileFormatException.getMessage().contains("Parquet")); + + Column nestedColumn = new Column("nested", new org.apache.doris.catalog.StructType( + new ArrayList<>(ImmutableList.of(new StructField("shape", geometry))))); + AnalysisException nestedException = Assert.assertThrows(AnalysisException.class, + () -> IcebergUtils.validateWriteSchema(ImmutableList.of(nestedColumn), 3, FileFormat.PARQUET)); + Assert.assertTrue(nestedException.getMessage().contains("nested in complex types")); + } + @Test public void testRejectVariantWritesWhenParquetShreddingIsEnabled() { String shredVariantsProperty = "write.parquet.shred-variants"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SpatialConstructorFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SpatialConstructorFunctionTest.java new file mode 100644 index 00000000000000..210353f9cfcc96 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SpatialConstructorFunctionTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.GeographyType; +import org.apache.doris.nereids.types.GeometryType; +import org.apache.doris.nereids.types.VarcharType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SpatialConstructorFunctionTest { + @Test + public void testLegacyGeomFromWkbReturnsVarchar() { + Assertions.assertEquals(VarcharType.SYSTEM_DEFAULT, + new StGeomFromWKB(new StringLiteral("0101000000000000000000F03F0000000000000040")) + .getDataType()); + Assertions.assertEquals(VarcharType.SYSTEM_DEFAULT, + new StGeometryFromWKB(new StringLiteral("0101000000000000000000F03F0000000000000040")) + .getDataType()); + } + + @Test + public void testTypedGeometryFromWkbReturnsDefaultGeometry() { + GeometryType defaultGeometry = new GeometryType("OGC:CRS84"); + Assertions.assertEquals(defaultGeometry, + new StGeometryFromWKBTyped( + new StringLiteral("0101000000000000000000F03F0000000000000040")) + .getDataType()); + } + + @Test + public void testGeogFromWkbReturnsDefaultGeography() { + GeographyType defaultGeography = new GeographyType("OGC:CRS84", "spherical"); + Assertions.assertEquals(defaultGeography, + new StGeogFromWKB(new StringLiteral("0101000000000000000000F03F0000000000000040")) + .getDataType()); + } + + @Test + public void testAsTextAcceptsGeometry() { + GeometryType defaultGeometry = new GeometryType("OGC:CRS84"); + StAstext asText = new StAstext(new StGeometryFromWKBTyped( + new StringLiteral("0101000000000000000000F03F0000000000000040"))); + Assertions.assertEquals(VarcharType.SYSTEM_DEFAULT, + asText.getDataType()); + Assertions.assertEquals(defaultGeometry, asText.expectedInputTypes().get(0)); + } + + @Test + public void testAsTextPreservesGeometryCrs() { + GeometryType webMercator = new GeometryType("EPSG:3857"); + StAstext asText = new StAstext(new SlotReference("spatial", webMercator)); + Assertions.assertEquals(VarcharType.SYSTEM_DEFAULT, asText.getDataType()); + Assertions.assertEquals(webMercator, asText.expectedInputTypes().get(0)); + } + + @Test + public void testAsTextPreservesGeographyMetadata() { + GeographyType geography = new GeographyType("OGC:CRS84", "vincenty"); + StAstext asText = new StAstext(new SlotReference("spatial", geography)); + Assertions.assertEquals(VarcharType.SYSTEM_DEFAULT, asText.getDataType()); + Assertions.assertEquals(geography, asText.expectedInputTypes().get(0)); + } + + @Test + public void testSpatialFunctionsPreserveParameterizedInputTypes() { + GeometryType geometry = new GeometryType("EPSG:3857"); + GeographyType geography = new GeographyType("OGC:CRS84", "vincenty"); + SlotReference geometrySlot = new SlotReference("geometry", geometry); + SlotReference geographySlot = new SlotReference("geography", geography); + + Assertions.assertEquals(geometry, new StAswkt(geometrySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geometry, new StAsBinary(geometrySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geometry, new StGeometryType(geometrySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geometry, new StX(geometrySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geometry, new StY(geometrySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geography, new StLength(geographySlot).expectedInputTypes().get(0)); + + Assertions.assertEquals(geometry, + new StDistance(geometrySlot, geographySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geography, + new StDistance(geometrySlot, geographySlot).expectedInputTypes().get(1)); + Assertions.assertEquals(geometry, + new StContains(geometrySlot, geographySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geometry, + new StIntersects(geometrySlot, geographySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geometry, + new StDisjoint(geometrySlot, geographySlot).expectedInputTypes().get(0)); + Assertions.assertEquals(geometry, + new StTouches(geometrySlot, geographySlot).expectedInputTypes().get(0)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java new file mode 100644 index 00000000000000..42bf379e8f9449 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.ColumnDef; +import org.apache.doris.analysis.TypeDef; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.GeographyType; +import org.apache.doris.nereids.types.GeometryType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Optional; + +class ColumnDefinitionTest { + private static final String INTERNAL_TABLE_SPATIAL_ERROR = + "GEOMETRY and GEOGRAPHY are not supported for Doris internal tables"; + + @Test + void rejectSpatialTypesForInternalTables() { + for (DataType type : new DataType[] {new GeometryType("EPSG:3857"), + new GeographyType("OGC:CRS84", "spherical")}) { + ColumnDefinition definition = + new ColumnDefinition("shape", type, false, null, true, Optional.empty(), ""); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> definition.validate(true, Collections.emptySet(), Collections.emptySet(), false, + KeysType.DUP_KEYS)); + Assertions.assertTrue(exception.getMessage().contains(INTERNAL_TABLE_SPATIAL_ERROR)); + } + } + + @Test + void rejectSpatialTypesForInternalTablesInLegacyPlanner() { + for (ScalarType type : new ScalarType[] {ScalarType.createGeometryType("EPSG:3857"), + ScalarType.createGeographyType("OGC:CRS84", "spherical")}) { + ColumnDef definition = new ColumnDef("shape", new TypeDef(type), false, null, true, + ColumnDef.DefaultValue.NOT_SET, ""); + org.apache.doris.common.AnalysisException exception = Assertions.assertThrows( + org.apache.doris.common.AnalysisException.class, () -> definition.analyze(true)); + Assertions.assertTrue(exception.getMessage().contains(INTERNAL_TABLE_SPATIAL_ERROR)); + } + } + + @Test + void rejectNestedSpatialTypesForInternalTables() { + ColumnDefinition nereidsDefinition = new ColumnDefinition("shapes", + ArrayType.of(new GeometryType("EPSG:3857")), false, null, true, Optional.empty(), ""); + AnalysisException nereidsException = Assertions.assertThrows(AnalysisException.class, + () -> nereidsDefinition.validate(true, Collections.emptySet(), Collections.emptySet(), false, + KeysType.DUP_KEYS)); + Assertions.assertTrue(nereidsException.getMessage().contains(INTERNAL_TABLE_SPATIAL_ERROR)); + + ColumnDef legacyDefinition = new ColumnDef("shapes", new TypeDef( + new org.apache.doris.catalog.ArrayType(ScalarType.createGeometryType("EPSG:3857"))), + false, null, true, ColumnDef.DefaultValue.NOT_SET, ""); + org.apache.doris.common.AnalysisException legacyException = Assertions.assertThrows( + org.apache.doris.common.AnalysisException.class, () -> legacyDefinition.analyze(true)); + Assertions.assertTrue(legacyException.getMessage().contains(INTERNAL_TABLE_SPATIAL_ERROR)); + } +} diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 1cb49602e60a76..144771ba173dd3 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -58,6 +58,10 @@ message PTypeNode { optional int32 variant_max_subcolumns_count = 6 [default = 0]; optional bool variant_enable_doc_mode = 7 [default = false]; optional bool variant_is_v2 = 8 [default = false]; + // Only set for Iceberg GEOMETRY/GEOGRAPHY types. + optional string spatial_crs = 9; + // Only set for Iceberg GEOGRAPHY type. + optional string spatial_algorithm = 10; }; // A flattened representation of a tree of column types obtained by depth-first diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index aecf7053e186b3..60495f71af8426 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -102,7 +102,9 @@ enum TPrimitiveType { UINT64 = 41, // only used in BE to represent offsets FIXED_LENGTH_OBJECT = 42 // only used in BE to represent fixed-length object VARBINARY = 43, // represent varbinary type - TIMESTAMPTZ = 44 // timestamp with time zone + TIMESTAMPTZ = 44, // timestamp with time zone + GEOMETRY = 45, + GEOGRAPHY = 46 } enum TTypeNodeType { @@ -148,6 +150,10 @@ struct TScalarType { 6: optional bool variant_enable_doc_mode = false; // Execution-only ColumnVariantV2 marker. Table metadata never sets this field. 7: optional bool variant_is_v2 = false; + // Only set for Iceberg GEOMETRY/GEOGRAPHY types. + 8: optional string spatial_crs; + // Only set for Iceberg GEOGRAPHY type. + 9: optional string spatial_algorithm; } // Represents a field in a STRUCT type. diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_spatial_v3.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_spatial_v3.groovy new file mode 100644 index 00000000000000..7d818dfe103be7 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_spatial_v3.groovy @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_spatial_v3", "p0,external,iceberg,external_docker,external_docker_iceberg,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_spatial_v3" + String dbName = "iceberg_spatial_v3_db" + String tableName = "spatial_values" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + def executeCommand = { String command, int timeoutSeconds = 300 -> + StringBuilder stdout = new StringBuilder() + StringBuilder stderr = new StringBuilder() + def process = new ProcessBuilder("/bin/bash", "-c", command).start() + process.consumeProcessOutput(stdout, stderr) + process.waitForOrKill(timeoutSeconds * 1000) + assertEquals(0, process.exitValue(), + "Command failed\nstdout:\n${stdout}\nstderr:\n${stderr}") + return stdout.toString() + } + + String dockerCommand = context.config.otherConfigs.get("externalDockerCommand") ?: "docker" + String sparkContainer = context.config.otherConfigs.get("icebergSparkContainer") + if (sparkContainer == null || sparkContainer.isEmpty()) { + String containers = executeCommand( + "${dockerCommand} ps --format '{{.ID}}\t{{.Names}}'", 30) + def matches = [] + containers.readLines().each { String line -> + String containerId = line.split(/\t/, 2)[0] + String probe = "${dockerCommand} exec ${containerId} bash -lc " + + "'test -f /mnt/SUCCESS && command -v spark-sql >/dev/null'" + try { + executeCommand(probe, 30) + matches.add(containerId) + } catch (Throwable ignored) { + // Only the Spark service has the Iceberg API used to create this V3 schema. + } + } + assertEquals(1, matches.size(), "Expected exactly one usable Spark Iceberg container") + sparkContainer = matches[0] + } + + String javaSource = ''' +import java.util.HashMap; +import java.util.Map; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Types; + +public class CreateIcebergSpatialV3Table { + public static void main(String[] args) { + Map properties = new HashMap<>(); + properties.put("type", "rest"); + properties.put("uri", "http://rest:8181"); + properties.put("warehouse", "s3://warehouse/wh/"); + properties.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO"); + properties.put("s3.endpoint", "http://minio:9000"); + properties.put("s3.path-style-access", "true"); + properties.put("s3.region", "us-east-1"); + Catalog catalog = CatalogUtil.buildIcebergCatalog("demo", properties, null); + SupportsNamespaces namespaceCatalog = (SupportsNamespaces) catalog; + Namespace namespace = Namespace.of(args[0]); + if (!namespaceCatalog.namespaceExists(namespace)) { + namespaceCatalog.createNamespace(namespace); + } + TableIdentifier identifier = TableIdentifier.of(namespace, args[1]); + if (catalog.tableExists(identifier)) { + catalog.dropTable(identifier); + } + Schema schema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "geom", Types.GeometryType.crs84()), + Types.NestedField.optional(3, "geog", Types.GeographyType.crs84())); + Map tableProperties = new HashMap<>(); + tableProperties.put("format-version", "3"); + tableProperties.put("write.format.default", "parquet"); + catalog.createTable(identifier, schema, PartitionSpec.unpartitioned(), tableProperties); + } +} +''' + String encodedJavaSource = javaSource.getBytes("UTF-8").encodeBase64().toString() + executeCommand("${dockerCommand} exec ${sparkContainer} bash -lc 'echo ${encodedJavaSource} " + + "| base64 -d >/tmp/CreateIcebergSpatialV3Table.java && " + + "javac -cp \"/opt/spark/jars/*\" /tmp/CreateIcebergSpatialV3Table.java && " + + "java -cp \"/tmp:/opt/spark/jars/*\" CreateIcebergSpatialV3Table ${dbName} ${tableName}'") + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type' = 'iceberg', + 'iceberg.catalog.type' = 'rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.region' = 'us-east-1' + ) + """ + + try { + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """ + insert into ${tableName} values + (1, ST_GeometryFromWKBTyped('0101000000000000000000F03F0000000000000040'), + ST_GeogFromWKB('0101000000000000000000F03F0000000000000040')) + """ + def rows = sql """ + select id, ST_AsText(geom), ST_AsText(geog) + from ${tableName} + order by id + """ + assertEquals(1, rows.size()) + assertEquals(1, rows[0][0].toString().toInteger()) + assertEquals("POINT (1 2)", rows[0][1].toString()) + assertEquals("POINT (1 2)", rows[0][2].toString()) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +}