From 9c384800dd4950117e9a06c7afe8c4b09e8f6c3f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 29 Jul 2026 15:49:02 +0800 Subject: [PATCH 1/4] [fix](iceberg) Enforce external write correctness (#66112) - Validate unsupported CTAS sinks before publishing table metadata, so setup failures cannot leave an unusable table or remove a concurrent replacement. - Preserve nullable string wrappers in truncate partition transforms. - Enforce Iceberg MERGE cardinality routing independently of `enable_strict_consistency_dml`. - Detect duplicate target matches with file-path-interned Roaring bitmaps and expose retained validation-state bytes in the sink profile. - Activate the negative regressions for the three retained Jira fixes and align the duplicate-match oracle with the emitted diagnostic. - Full build: `./build.sh --fe --be -j 48` - FE: `CreateTableCommandTest`, `IcebergDDLAndDMLPlanTest` (19 tests) - BE: `VIcebergMergeSinkTest` (8 tests, including 100,000 matched rows) - Regression: `test_paimon_ctas_atomicity_negative` (1 suite, passed) - Iceberg end-to-end SQL: duplicate-source MERGE rejected atomically; nullable/merge truncate writes and logical rows verified; physical partition values verified from the same Iceberg tables through Spark metadata. The Doris-side `$partitions`/`$snapshots` checks in the Iceberg suites currently hit an unrelated JNI scanner initialization abort on the current master test binary; the equivalent table state and metadata assertions above passed. (cherry picked from commit 8305fe71867fb2acab5a769fe9a098ada59aff50) --- be/src/agent/be_exec_version_manager.cpp | 4 +- be/src/agent/be_exec_version_manager.h | 1 + be/src/exec/sink/viceberg_delete_sink.cpp | 53 +++- be/src/exec/sink/viceberg_delete_sink.h | 8 + be/src/exec/sink/viceberg_merge_sink.cpp | 142 ++++++++- be/src/exec/sink/viceberg_merge_sink.h | 7 + .../writer/iceberg/partition_transformers.h | 5 +- .../iceberg/viceberg_delete_file_writer.h | 2 + .../iceberg/viceberg_partition_writer.cpp | 31 +- .../iceberg/viceberg_partition_writer.h | 9 +- .../writer/iceberg/viceberg_table_writer.cpp | 33 ++- .../writer/iceberg/viceberg_table_writer.h | 12 + .../exec/sink/viceberg_merge_sink_test.cpp | 275 +++++++++++++++++- .../iceberg/partition_transformers_test.cpp | 26 ++ .../java/org/apache/doris/common/Config.java | 2 +- .../datasource/paimon/PaimonMetadataOps.java | 27 +- .../translator/PhysicalPlanTranslator.java | 3 +- .../properties/RequestPropertyDeriver.java | 5 +- ...rgMergeSinkToPhysicalIcebergMergeSink.java | 1 + .../plans/commands/CreateTableCommand.java | 35 ++- .../plans/commands/IcebergMergeCommand.java | 1 + .../plans/commands/IcebergUpdateCommand.java | 1 + .../logical/LogicalIcebergMergeSink.java | 26 +- .../physical/PhysicalIcebergMergeSink.java | 40 ++- .../doris/planner/IcebergMergeSink.java | 7 +- .../iceberg/IcebergDDLAndDMLPlanTest.java | 149 +++++++++- .../paimon/PaimonMetadataOpsTest.java | 64 +++- .../commands/CreateTableCommandTest.java | 133 +++++++++ .../insert/IcebergMergeExecutorTest.java | 3 +- .../doris/planner/IcebergMergeSinkTest.java | 18 +- gensrc/thrift/DataSinks.thrift | 2 + ..._write_merge_duplicate_source_negative.out | 2 + .../PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md | 9 +- .../write/ICEBERG_WRITE_P0_COVERAGE.md | 16 +- ...ite_merge_duplicate_source_negative.groovy | 93 +++++- ...eberg_write_merge_truncate_negative.groovy | 15 +- ...rg_write_nullable_truncate_negative.groovy | 8 - ...test_paimon_ctas_atomicity_negative.groovy | 30 +- 38 files changed, 1176 insertions(+), 122 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java diff --git a/be/src/agent/be_exec_version_manager.cpp b/be/src/agent/be_exec_version_manager.cpp index 9ad9d9e9a5949a..4c3bdbd9f31445 100644 --- a/be/src/agent/be_exec_version_manager.cpp +++ b/be/src/agent/be_exec_version_manager.cpp @@ -124,8 +124,10 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers // 10: start from doris 4.0.3 // a. use new fixed object serialization way. +// 11: start from master +// a. enforce Iceberg SQL MERGE cardinality only when every executing BE supports it. -const int BeExecVersionManager::max_be_exec_version = 10; +const int BeExecVersionManager::max_be_exec_version = 11; const int BeExecVersionManager::min_be_exec_version = 0; std::map> BeExecVersionManager::_function_change_map {}; std::set BeExecVersionManager::_function_restrict_map; diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h index 7ced5b9749ed5a..f5c16e9a2a55ef 100644 --- a/be/src/agent/be_exec_version_manager.h +++ b/be/src/agent/be_exec_version_manager.h @@ -33,6 +33,7 @@ constexpr inline int AGGREGATION_2_1_VERSION = constexpr inline int USE_CONST_SERDE = 8; // support const column in serialize/deserialize function: PR #41175 constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10; +constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11; class BeExecVersionManager { public: diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index ef167ecdbcaea3..172fdd28177c62 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -40,6 +40,7 @@ #include "format/transformer/vfile_format_transformer.h" #include "io/file_factory.h" #include "runtime/runtime_state.h" +#include "util/debug_points.h" #include "util/slice.h" #include "util/string_util.h" #include "util/uid_util.h" @@ -252,17 +253,26 @@ Status VIcebergDeleteSink::close(Status close_status) { if (!close_status.ok()) { LOG(WARNING) << fmt::format("VIcebergDeleteSink close with error: {}", close_status.to_string()); + _cleanup_created_files(); return close_status; } + Status write_status = Status::OK(); + DBUG_EXECUTE_IF("VIcebergDeleteSink.close.inject_failure", { + write_status = Status::InternalError("injected Iceberg delete close failure"); + }); + if (_delete_type == TFileContent::POSITION_DELETES && !_file_deletions.empty()) { SCOPED_TIMER(_write_delete_files_timer); - if (_format_version >= 3) { - RETURN_IF_ERROR(_write_deletion_vector_files(_file_deletions)); - } else { - RETURN_IF_ERROR(_write_position_delete_files(_file_deletions)); + if (write_status.ok()) { + write_status = _format_version >= 3 ? _write_deletion_vector_files(_file_deletions) + : _write_position_delete_files(_file_deletions); } } + if (!write_status.ok()) { + _cleanup_created_files(); + return write_status; + } // Update counters if (_delete_file_count_counter) { @@ -278,9 +288,33 @@ Status VIcebergDeleteSink::close(Status close_status) { } } + if (!_defer_file_cleanup_until_outer_close) { + _created_files.clear(); + } + return Status::OK(); } +void VIcebergDeleteSink::finish_deferred_file_cleanup(Status outer_status) { + if (!outer_status.ok()) { + _cleanup_created_files(); + } else { + _created_files.clear(); + } + _defer_file_cleanup_until_outer_close = false; +} + +void VIcebergDeleteSink::_cleanup_created_files() { + for (const auto& [fs, path] : _created_files) { + Status delete_status = fs->delete_file(path); + if (!delete_status.ok() && !delete_status.is()) { + LOG(WARNING) << fmt::format("Failed to delete Iceberg delete file {}: {}", path, + delete_status.to_string()); + } + } + _created_files.clear(); +} + int VIcebergDeleteSink::_get_row_id_column_index(const Block& block) { // Find __DORIS_ICEBERG_ROWID_COL__ column in block for (size_t i = 0; i < block.columns(); ++i) { @@ -448,9 +482,13 @@ Status VIcebergDeleteSink::_write_position_delete_files( } // Open writer - RETURN_IF_ERROR(writer->open(_state, _state->runtime_profile(), - _position_delete_output_expr_ctxs, column_names, _hadoop_conf, - _file_type, _broker_addresses)); + Status open_status = + writer->open(_state, _state->runtime_profile(), _position_delete_output_expr_ctxs, + column_names, _hadoop_conf, _file_type, _broker_addresses); + if (writer->file_system() != nullptr) { + _created_files.emplace_back(writer->file_system(), delete_file_path); + } + RETURN_IF_ERROR(open_status); // Build block with (file_path, pos) columns std::vector positions; @@ -669,6 +707,7 @@ Status VIcebergDeleteSink::_write_puffin_file(const std::string& puffin_path, auto fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description)); io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false}; io::FileWriterPtr file_writer; + _created_files.emplace_back(fs, puffin_path); RETURN_IF_ERROR(fs->create_file(file_description.path, &file_writer, &file_writer_options)); constexpr char PUFFIN_MAGIC[] = {'\x50', '\x46', '\x41', '\x31'}; diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h index 647ee92f525733..55698ae0404b14 100644 --- a/be/src/exec/sink/viceberg_delete_sink.h +++ b/be/src/exec/sink/viceberg_delete_sink.h @@ -77,6 +77,10 @@ class VIcebergDeleteSink final : public AsyncResultWriter { Status close(Status) override; + void defer_file_cleanup_until_outer_close() { _defer_file_cleanup_until_outer_close = true; } + + void finish_deferred_file_cleanup(Status outer_status); + private: /** * Extract $row_id column from block and group by file_path. @@ -129,6 +133,7 @@ class VIcebergDeleteSink final : public AsyncResultWriter { const std::vector& positions, Block& output_block); Status _init_position_delete_output_exprs(); std::string _get_file_extension() const; + void _cleanup_created_files(); TDataSink _t_sink; RuntimeState* _state = nullptr; @@ -141,6 +146,9 @@ class VIcebergDeleteSink final : public AsyncResultWriter { // Collected commit data from all writers std::vector _commit_data_list; + // MERGE owns both data and delete objects until both inner sinks close successfully. + std::vector, std::string>> _created_files; + bool _defer_file_cleanup_until_outer_close = false; // TODO: All deletions are held in memory until close(). Consider flushing // per-file when the upstream guarantees file_path ordering, or flushing // when estimated memory exceeds a threshold, to reduce peak memory usage. diff --git a/be/src/exec/sink/viceberg_merge_sink.cpp b/be/src/exec/sink/viceberg_merge_sink.cpp index e9f4d6bf5c655a..5ff5a0a1f28150 100644 --- a/be/src/exec/sink/viceberg_merge_sink.cpp +++ b/be/src/exec/sink/viceberg_merge_sink.cpp @@ -19,12 +19,17 @@ #include +#include "agent/be_exec_version_manager.h" #include "common/consts.h" #include "common/exception.h" #include "common/logging.h" #include "core/block/block.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "exec/sink/sink_common.h" #include "exec/sink/viceberg_delete_sink.h" #include "exec/sink/writer/iceberg/viceberg_table_writer.h" @@ -52,8 +57,10 @@ Status VIcebergMergeSink::init_properties(ObjectPool* pool, const RowDescriptor& _table_writer = std::make_unique(_table_sink, _table_output_expr_ctxs, nullptr, nullptr); + _table_writer->defer_file_cleanup_until_outer_close(); _delete_writer = std::make_unique(_delete_sink, _delete_output_expr_ctxs, nullptr, nullptr); + _delete_writer->defer_file_cleanup_until_outer_close(); RETURN_IF_ERROR(_table_writer->init_properties(pool, row_desc)); RETURN_IF_ERROR(_delete_writer->init_properties(pool)); return Status::OK(); @@ -65,6 +72,14 @@ Status VIcebergMergeSink::open(RuntimeState* state, RuntimeProfile* profile) { _written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT); _insert_rows_counter = ADD_COUNTER(profile, "InsertRows", TUnit::UNIT); _delete_rows_counter = ADD_COUNTER(profile, "DeleteRows", TUnit::UNIT); + // The query-wide version keeps validation all-or-nothing during a rolling BE upgrade. + _require_merge_cardinality_check = + _require_merge_cardinality_check && + state->be_exec_version() >= SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION; + if (_require_merge_cardinality_check) { + _matched_row_id_state_bytes_counter = + ADD_COUNTER(profile, "MatchedRowIdStateBytes", TUnit::BYTES); + } _send_data_timer = ADD_TIMER(profile, "SendDataTime"); _open_timer = ADD_TIMER(profile, "OpenTime"); _close_timer = ADD_TIMER(profile, "CloseTime"); @@ -94,8 +109,6 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) { return Status::OK(); } - _row_count += output_block.rows(); - if (_operation_idx < 0 || _row_id_idx < 0) { return Status::InternalError("Iceberg merge sink missing operation/row_id columns"); } @@ -120,17 +133,26 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) { if (delete_op) { delete_filter[i] = 1; has_delete = true; - ++_delete_row_count; ++delete_rows; } if (insert_op) { insert_filter[i] = 1; has_insert = true; - ++_insert_row_count; ++insert_rows; } } + if (_require_merge_cardinality_check) { + // The physical sink hashes matched rows by row_id, so exact state retained across blocks + // enforces SQL MERGE cardinality for the whole query without changing UPDATE semantics. + RETURN_IF_ERROR(_validate_matched_row_ids(output_block, delete_filter.data())); + COUNTER_SET(_matched_row_id_state_bytes_counter, + static_cast(_matched_row_id_state_size)); + } + _row_count += output_block.rows(); + _delete_row_count += delete_rows; + _insert_row_count += insert_rows; + bool skip_io = false; #ifdef BE_TEST skip_io = _skip_io; @@ -167,6 +189,105 @@ Status VIcebergMergeSink::write(RuntimeState* state, Block& block) { return Status::OK(); } +Status VIcebergMergeSink::_validate_matched_row_ids(const Block& block, + const uint8_t* delete_filter) { + const auto& row_id = block.get_by_position(_row_id_idx); + const IColumn* row_id_data = row_id.column.get(); + const IDataType* row_id_type = row_id.type.get(); + const auto* nullable_row_id = check_and_get_column(row_id_data); + if (nullable_row_id != nullptr) { + row_id_data = nullable_row_id->get_nested_column_ptr().get(); + } + if (const auto* nullable_type = check_and_get_data_type(row_id_type)) { + row_id_type = nullable_type->get_nested_type().get(); + } + + const auto* struct_column = check_and_get_column(row_id_data); + const auto* struct_type = check_and_get_data_type(row_id_type); + if (struct_column == nullptr || struct_type == nullptr) { + return Status::InternalError("Iceberg merge row_id column is not a struct"); + } + + int file_path_idx = -1; + int row_position_idx = -1; + const auto& field_names = struct_type->get_element_names(); + for (size_t i = 0; i < field_names.size(); ++i) { + std::string field_name = doris::to_lower(field_names[i]); + if (field_name == "file_path") { + file_path_idx = static_cast(i); + } else if (field_name == "row_position") { + row_position_idx = static_cast(i); + } + } + if (file_path_idx < 0 || row_position_idx < 0) { + return Status::InternalError( + "Iceberg merge row_id must contain file_path and row_position fields"); + } + + const auto& file_path_column = struct_column->get_column_ptr(file_path_idx); + const auto& row_position_column = struct_column->get_column_ptr(row_position_idx); + const auto* nullable_file_path = check_and_get_column(file_path_column.get()); + const auto* nullable_row_position = + check_and_get_column(row_position_column.get()); + const auto* file_paths = + check_and_get_column(remove_nullable(file_path_column).get()); + const auto* row_positions = check_and_get_column>( + remove_nullable(row_position_column).get()); + if (file_paths == nullptr || row_positions == nullptr) { + return Status::InternalError("Iceberg merge row_id fields have incorrect types"); + } + + std::map touched_bitmap_sizes; + for (size_t i = 0; i < block.rows(); ++i) { + if (delete_filter[i] == 0) { + continue; + } + if ((nullable_row_id != nullptr && nullable_row_id->is_null_at(i)) || + (nullable_file_path != nullptr && nullable_file_path->is_null_at(i)) || + (nullable_row_position != nullptr && nullable_row_position->is_null_at(i))) { + return Status::InternalError("Iceberg merge matched row_id cannot be null"); + } + + int64_t row_position = row_positions->get_element(i); + if (row_position < 0) { + return Status::InternalError("Invalid row_position {} in Iceberg merge row_id", + row_position); + } + // Intern each file path once and keep exact positions in a compressed bitmap; retaining a + // full path string per matched row makes MERGE memory grow with path_length * row_count. + auto [file_it, inserted] = + _matched_row_positions.try_emplace(file_paths->get_data_at(i).to_string()); + auto* positions = &file_it->second; + auto touched_it = touched_bitmap_sizes.find(positions); + if (touched_it == touched_bitmap_sizes.end()) { + touched_it = touched_bitmap_sizes.emplace(positions, positions->getSizeInBytes()).first; + } + if (inserted) { + _matched_row_id_state_size += + sizeof(std::pair); + _matched_row_id_state_size += file_it->first.capacity(); + _matched_row_id_state_size += touched_it->second; + } + if (!positions->addChecked(static_cast(row_position))) { + return Status::InvalidArgument( + "Iceberg MERGE failed because multiple source rows matched the same target " + "row"); + } + } + + // Measure only bitmaps touched by this block; rescanning all retained files on every write + // makes a many-file MERGE quadratic in the number of input blocks. + for (const auto& [positions, previous_size] : touched_bitmap_sizes) { + size_t current_size = positions->getSizeInBytes(); + if (current_size >= previous_size) { + _matched_row_id_state_size += current_size - previous_size; + } else { + _matched_row_id_state_size -= previous_size - current_size; + } + } + return Status::OK(); +} + Status VIcebergMergeSink::close(Status close_status) { SCOPED_TIMER(_close_timer); @@ -201,10 +322,14 @@ Status VIcebergMergeSink::close(Status close_status) { COUNTER_SET(_delete_rows_counter, static_cast(_delete_row_count)); } - if (!table_status.ok()) { - return table_status; + Status result_status = table_status.ok() ? delete_status : table_status; + if (_table_writer) { + _table_writer->finish_deferred_file_cleanup(result_status); + } + if (_delete_writer) { + _delete_writer->finish_deferred_file_cleanup(result_status); } - return delete_status; + return result_status; } Status VIcebergMergeSink::_build_inner_sinks() { @@ -213,6 +338,9 @@ Status VIcebergMergeSink::_build_inner_sinks() { } const auto& merge_sink = _t_sink.iceberg_merge_sink; + // Missing means an old FE plan, which predates SQL MERGE cardinality validation. + _require_merge_cardinality_check = merge_sink.__isset.require_merge_cardinality_check && + merge_sink.require_merge_cardinality_check; TIcebergTableSink table_sink; if (merge_sink.__isset.db_name) { diff --git a/be/src/exec/sink/viceberg_merge_sink.h b/be/src/exec/sink/viceberg_merge_sink.h index d7f87b87012149..f3733a3318230e 100644 --- a/be/src/exec/sink/viceberg_merge_sink.h +++ b/be/src/exec/sink/viceberg_merge_sink.h @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -26,6 +27,7 @@ #include "common/status.h" #include "exec/sink/writer/async_result_writer.h" #include "exprs/vexpr_fwd.h" +#include "roaring/roaring64map.hh" #include "runtime/descriptors.h" #include "runtime/runtime_profile.h" @@ -60,6 +62,7 @@ class VIcebergMergeSink final : public AsyncResultWriter { private: Status _build_inner_sinks(); Status _prepare_output_layout(); + Status _validate_matched_row_ids(const Block& block, const uint8_t* delete_filter); TDataSink _t_sink; TDataSink _table_sink; @@ -73,6 +76,9 @@ class VIcebergMergeSink final : public AsyncResultWriter { int _operation_idx = -1; int _row_id_idx = -1; std::vector _data_column_indices; + std::map _matched_row_positions; + size_t _matched_row_id_state_size = sizeof(std::map); + bool _require_merge_cardinality_check = false; VExprContextSPtrs _table_output_expr_ctxs; VExprContextSPtrs _delete_output_expr_ctxs; @@ -84,6 +90,7 @@ class VIcebergMergeSink final : public AsyncResultWriter { RuntimeProfile::Counter* _written_rows_counter = nullptr; RuntimeProfile::Counter* _insert_rows_counter = nullptr; RuntimeProfile::Counter* _delete_rows_counter = nullptr; + RuntimeProfile::Counter* _matched_row_id_state_bytes_counter = nullptr; RuntimeProfile::Counter* _send_data_timer = nullptr; RuntimeProfile::Counter* _open_timer = nullptr; RuntimeProfile::Counter* _close_timer = nullptr; diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.h b/be/src/exec/sink/writer/iceberg/partition_transformers.h index 6ca8ef11fd4c40..c89b1641ca307f 100644 --- a/be/src/exec/sink/writer/iceberg/partition_transformers.h +++ b/be/src/exec/sink/writer/iceberg/partition_transformers.h @@ -167,7 +167,10 @@ class StringTruncatePartitionColumnTransform : public PartitionColumnTransform { // Create a temp_block to execute substring function. Block temp_block; - temp_block.insert(column_with_type_and_name); + // Substring requires the physical ColumnString; preserve nullability separately and + // restore the original null map after transforming the nested values. + temp_block.insert({string_column_ptr, remove_nullable(column_with_type_and_name.type), + column_with_type_and_name.name}); temp_block.insert({int_type->create_column_const(temp_block.rows(), to_field(1)), int_type, "const 1"}); temp_block.insert( diff --git a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h index c242731dc1548a..4ceb7080a3c244 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h @@ -94,6 +94,8 @@ class VIcebergDeleteFileWriter { */ int64_t get_file_size() const { return _file_size; } + std::shared_ptr file_system() const { return _fs; } + private: TFileContent::type _delete_type; std::string _output_path; diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index 25da8724b2899b..ba7644daec751f 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -36,7 +36,8 @@ VIcebergPartitionWriter::VIcebergPartitionWriter( const std::string* iceberg_schema_json, std::vector write_column_names, WriteInfo write_info, std::string file_name, int file_name_index, TFileFormatType::type file_format_type, TFileCompressType::type compress_type, - const std::map& hadoop_conf) + const std::map& hadoop_conf, + ClosedFileCallback closed_file_callback) : _partition_values(std::move(partition_values)), _write_output_expr_ctxs(write_output_expr_ctxs), _schema(schema), @@ -47,7 +48,8 @@ VIcebergPartitionWriter::VIcebergPartitionWriter( _file_name_index(file_name_index), _file_format_type(file_format_type), _compress_type(compress_type), - _hadoop_conf(hadoop_conf) { + _hadoop_conf(hadoop_conf), + _closed_file_callback(std::move(closed_file_callback)) { if (t_sink.iceberg_table_sink.__isset.collect_column_stats) { _collect_column_stats = t_sink.iceberg_table_sink.collect_column_stats; } @@ -61,9 +63,8 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil if (!_write_info.broker_addresses.empty()) { fs_properties.broker_addresses = &(_write_info.broker_addresses); } - io::FileDescription file_description = { - .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()), - .fs_name {}}; + _path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()); + io::FileDescription file_description = {.path = _path, .fs_name {}}; _fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description)); io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false}; RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options)); @@ -128,16 +129,28 @@ Status VIcebergPartitionWriter::close(const Status& status) { } bool status_ok = result_status.ok() && status.ok(); if (!status_ok && _fs != nullptr) { - auto path = fmt::format("{}/{}", _write_info.write_path, _file_name); - Status st = _fs->delete_file(path); + // delete the actual created file, otherwise an orphan file is left behind + Status st = _fs->delete_file(_path); if (!st.ok()) { - LOG(WARNING) << fmt::format("Delete file {} failed, reason: {}", path, st.to_string()); + LOG(WARNING) << fmt::format("Delete file {} failed, reason: {}", _path, st.to_string()); } } if (status_ok) { TIcebergCommitData commit_data; - RETURN_IF_ERROR(_build_iceberg_commit_data(&commit_data)); + Status commit_status = _build_iceberg_commit_data(&commit_data); + if (!commit_status.ok()) { + // A closed object without commit data can never be published, so remove it immediately. + Status delete_status = _fs->delete_file(_path); + if (!delete_status.ok()) { + LOG(WARNING) << fmt::format("Delete file {} failed, reason: {}", _path, + delete_status.to_string()); + } + return commit_status; + } _state->add_iceberg_commit_datas(commit_data); + if (_closed_file_callback) { + _closed_file_callback(_fs, _path); + } } return result_status; } diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h index b0839a82ed3796..f3a7a6002f846c 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h @@ -19,6 +19,8 @@ #include +#include + #include "exec/sink/writer/iceberg/vpartition_writer_base.h" #include "exprs/vexpr_fwd.h" #include "format/table/iceberg/schema.h" @@ -43,6 +45,9 @@ class VFileFormatTransformer; class VIcebergPartitionWriter : public IPartitionWriterBase { public: + using ClosedFileCallback = + std::function, const std::string&)>; + VIcebergPartitionWriter(const TDataSink& t_sink, std::vector partition_values, const VExprContextSPtrs& write_output_expr_ctxs, const doris::iceberg::Schema& schema, @@ -51,7 +56,8 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { std::string file_name, int file_name_index, TFileFormatType::type file_format_type, TFileCompressType::type compress_type, - const std::map& hadoop_conf); + const std::map& hadoop_conf, + ClosedFileCallback closed_file_callback = nullptr); Status open(RuntimeState* state, RuntimeProfile* profile, const RowDescriptor* row_desc) override; @@ -93,6 +99,7 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { TFileFormatType::type _file_format_type; TFileCompressType::type _compress_type; const std::map& _hadoop_conf; + ClosedFileCallback _closed_file_callback; bool _collect_column_stats = true; std::shared_ptr _fs = nullptr; diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index dc5f25221220ad..b9eeeac38a7a30 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -33,6 +33,7 @@ #include "exprs/vexpr_context.h" #include "format/table/iceberg/partition_spec_parser.h" #include "format/table/iceberg/schema_parser.h" +#include "io/fs/file_system.h" #include "runtime/runtime_state.h" namespace doris { @@ -486,9 +487,36 @@ Status VIcebergTableWriter::close(Status status) { COUNTER_SET(_close_timer, _close_ns); COUNTER_SET(_write_file_counter, _write_file_count); } + if (!status.ok() || !result_status.ok()) { + _cleanup_closed_files(); + } else if (!_defer_file_cleanup_until_outer_close) { + _closed_files.clear(); + } return result_status; } +void VIcebergTableWriter::finish_deferred_file_cleanup(Status outer_status) { + // A successful inner close does not publish MERGE files; the sibling delete close still + // decides whether these data objects must be removed or released to the transaction. + if (!outer_status.ok()) { + _cleanup_closed_files(); + } else { + _closed_files.clear(); + } + _defer_file_cleanup_until_outer_close = false; +} + +void VIcebergTableWriter::_cleanup_closed_files() { + for (const auto& [fs, path] : _closed_files) { + Status delete_status = fs->delete_file(path); + if (!delete_status.ok()) { + LOG(WARNING) << fmt::format("Delete rolled Iceberg file {} failed, reason: {}", path, + delete_status.to_string()); + } + } + _closed_files.clear(); +} + std::string VIcebergTableWriter::_partition_to_path(const doris::iceberg::StructLike& data) { std::stringstream ss; for (size_t i = 0; i < _iceberg_partition_columns.size(); i++) { @@ -612,7 +640,10 @@ std::shared_ptr VIcebergTableWriter::_create_partition_wri &_t_sink.iceberg_table_sink.schema_json, column_names, write_info, (file_name == nullptr) ? _compute_file_name() : *file_name, file_name_index, iceberg_table_sink.file_format, iceberg_table_sink.compression_type, - iceberg_table_sink.hadoop_config); + iceberg_table_sink.hadoop_config, + [this](std::shared_ptr fs, const std::string& path) { + _closed_files.emplace_back(std::move(fs), path); + }); }; auto partition_write = create_writer_lambda(file_name, file_name_index); if (iceberg_table_sink.__isset.sort_info) { diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h index cc7cec1fdad880..070019d85db9be 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h @@ -32,6 +32,10 @@ namespace doris { +namespace io { +class FileSystem; +} + class ObjectPool; class RuntimeState; @@ -60,6 +64,10 @@ class VIcebergTableWriter final : public AsyncResultWriter { Status close(Status) override; + void defer_file_cleanup_until_outer_close() { _defer_file_cleanup_until_outer_close = true; } + + void finish_deferred_file_cleanup(Status outer_status); + bool is_rewrite_compaction() const { return _write_type == TIcebergWriteType::REWRITE; } TIcebergWriteType::type write_type() const { return _write_type; } @@ -137,6 +145,7 @@ class VIcebergTableWriter final : public AsyncResultWriter { Status _write_prepared_block(Block& output_block); Status _process_row_lineage_columns(Block& block); + void _cleanup_closed_files(); // Currently it is a copy, maybe it is better to use move semantics to eliminate it. TDataSink _t_sink; @@ -171,6 +180,9 @@ class VIcebergTableWriter final : public AsyncResultWriter { std::vector _static_partition_value_list; std::unordered_map> _partitions_to_writers; + // Rolled writers are no longer active, so retain their physical paths until statement outcome is known. + std::vector, std::string>> _closed_files; + bool _defer_file_cleanup_until_outer_close = false; VExprContextSPtrs _write_output_vexpr_ctxs; size_t _row_count = 0; const RowDescriptor* _row_desc = nullptr; diff --git a/be/test/exec/sink/viceberg_merge_sink_test.cpp b/be/test/exec/sink/viceberg_merge_sink_test.cpp index d5c9ac1b891440..eb7c0159d5fd38 100644 --- a/be/test/exec/sink/viceberg_merge_sink_test.cpp +++ b/be/test/exec/sink/viceberg_merge_sink_test.cpp @@ -19,6 +19,11 @@ #include +#include +#include + +#include "agent/be_exec_version_manager.h" +#include "common/config.h" #include "common/consts.h" #include "common/object_pool.h" #include "core/block/block.h" @@ -28,13 +33,17 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "exec/sink/viceberg_delete_sink.h" +#include "exec/sink/writer/iceberg/viceberg_table_writer.h" #include "exprs/vexpr_context.h" #include "gen_cpp/DataSinks_types.h" #include "gen_cpp/Types_types.h" +#include "io/fs/local_file_system.h" #include "runtime/runtime_profile.h" #include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_slot_ref.h" +#include "util/debug_points.h" namespace doris { @@ -47,7 +56,7 @@ class VIcebergMergeSinkTest : public testing::Test { "]}"; } - TDataSink build_sink() { + TDataSink build_sink(bool require_cardinality_check = true, bool set_cardinality_check = true) { TDataSink t_sink; t_sink.__set_type(TDataSinkType::ICEBERG_MERGE_SINK); @@ -64,6 +73,9 @@ class VIcebergMergeSinkTest : public testing::Test { merge_sink.__set_file_type(TFileType::FILE_LOCAL); merge_sink.__set_delete_type(TFileContent::POSITION_DELETES); merge_sink.__set_partition_spec_id_for_delete(0); + if (set_cardinality_check) { + merge_sink.__set_require_merge_cardinality_check(require_cardinality_check); + } t_sink.__set_iceberg_merge_sink(merge_sink); return t_sink; @@ -106,7 +118,8 @@ class VIcebergMergeSinkTest : public testing::Test { return output_exprs; } - Block build_block_with_ops(const std::vector& ops) { + Block build_block_with_ops(const std::vector& ops, bool distinct_files = true, + size_t file_index_offset = 0) { Block block; auto op_col = ColumnInt8::create(); @@ -121,9 +134,11 @@ class VIcebergMergeSinkTest : public testing::Test { auto id_col = ColumnInt32::create(); auto name_col = ColumnString::create(); for (size_t i = 0; i < ops.size(); ++i) { - std::string file_path = "file" + std::to_string(i + 1) + ".parquet"; + std::string file_path = + distinct_files ? "file" + std::to_string(file_index_offset + i + 1) + ".parquet" + : "shared-file.parquet"; file_path_col->insert_data(file_path.data(), file_path.size()); - row_pos_col->insert_value(static_cast((i + 1) * 10)); + row_pos_col->insert_value(static_cast((file_index_offset + i + 1) * 10)); id_col->insert_value(static_cast(i + 1)); char name_value = static_cast('a' + i); name_col->insert_data(&name_value, 1); @@ -315,4 +330,256 @@ TEST_F(VIcebergMergeSinkTest, TestSchemaMismatch) { ASSERT_NE(std::string::npos, status.to_string().find("do not match schema columns")); } +TEST_F(VIcebergMergeSinkTest, TestRejectsDuplicateMatchedTargetAcrossBlocks) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = std::make_shared(build_sink(), output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("iceberg_merge_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + + Block first = build_block_with_ops({3}); + ASSERT_TRUE(sink->write(&state, first).ok()); + Block duplicate = build_block_with_ops({3}); + Status status = sink->write(&state, duplicate); + + ASSERT_FALSE(status.ok()); + ASSERT_NE(std::string::npos, + status.to_string().find("multiple source rows matched the same target row")); +} + +TEST_F(VIcebergMergeSinkTest, TestUpdateSkipsCardinalityState) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = + std::make_shared(build_sink(false), output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("iceberg_update_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + + Block first = build_block_with_ops({3}); + ASSERT_TRUE(sink->write(&state, first).ok()); + Block duplicate = build_block_with_ops({3}); + ASSERT_TRUE(sink->write(&state, duplicate).ok()); + EXPECT_TRUE(sink->_matched_row_positions.empty()); + EXPECT_EQ(nullptr, profile.get_counter("MatchedRowIdStateBytes")); +} + +TEST_F(VIcebergMergeSinkTest, TestOldFePlanSkipsCardinalityState) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = std::make_shared(build_sink(false, false), output_exprs, nullptr, + nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("old_fe_iceberg_update_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + Block duplicate = build_block_with_ops({3, 3}, false); + ASSERT_TRUE(sink->write(&state, duplicate).ok()); + EXPECT_TRUE(sink->_matched_row_positions.empty()); +} + +TEST_F(VIcebergMergeSinkTest, TestRollingUpgradeSkipsCardinalityState) { + ObjectPool pool; + MockRuntimeState state; + state.set_be_exec_version(SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION - 1); + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = std::make_shared(build_sink(), output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("rolling_upgrade_iceberg_merge_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + Block duplicate = build_block_with_ops({3, 3}, false); + ASSERT_TRUE(sink->write(&state, duplicate).ok()); + EXPECT_TRUE(sink->_matched_row_positions.empty()); +} + +TEST_F(VIcebergMergeSinkTest, TestErrorCloseRemovesRolledDataFiles) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = std::make_shared(build_sink(), output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("iceberg_merge_cleanup_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + + std::filesystem::path path = + std::filesystem::temp_directory_path() / "doris_iceberg_merge_rolled_file.parquet"; + { + std::ofstream output(path); + output << "rolled-data"; + } + ASSERT_TRUE(std::filesystem::exists(path)); + sink->_table_writer->_closed_files.emplace_back(io::global_local_filesystem(), path.string()); + + Status failure = Status::InvalidArgument("late duplicate"); + EXPECT_FALSE(sink->close(failure).ok()); + EXPECT_FALSE(std::filesystem::exists(path)); +} + +TEST_F(VIcebergMergeSinkTest, TestDeleteCloseFailureRemovesBothInnerSinkFiles) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = std::make_shared(build_sink(), output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("iceberg_merge_delete_close_cleanup_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + + std::filesystem::path data_path = std::filesystem::temp_directory_path() / + "doris_iceberg_merge_delete_close_data.parquet"; + std::filesystem::path delete_path = std::filesystem::temp_directory_path() / + "doris_iceberg_merge_delete_close_position.parquet"; + { + std::ofstream output(data_path); + output << "closed-data"; + } + { + std::ofstream output(delete_path); + output << "closed-delete"; + } + ASSERT_TRUE(std::filesystem::exists(data_path)); + ASSERT_TRUE(std::filesystem::exists(delete_path)); + sink->_table_writer->_closed_files.emplace_back(io::global_local_filesystem(), + data_path.string()); + sink->_delete_writer->_created_files.emplace_back(io::global_local_filesystem(), + delete_path.string()); + + bool previous_enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + DebugPoints::instance()->add("VIcebergDeleteSink.close.inject_failure"); + Status status = sink->close(Status::OK()); + DebugPoints::instance()->clear(); + config::enable_debug_points = previous_enable_debug_points; + + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("injected Iceberg delete close failure")); + EXPECT_FALSE(std::filesystem::exists(data_path)); + EXPECT_FALSE(std::filesystem::exists(delete_path)); +} + +TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdsUseCompactRetainedState) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = std::make_shared(build_sink(), output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("iceberg_merge_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + + constexpr size_t row_count = 100000; + std::vector operations(row_count, 3); + Block block = build_block_with_ops(operations, false); + ASSERT_TRUE(sink->write(&state, block).ok()); + + auto* retained_bytes = profile.get_counter("MatchedRowIdStateBytes"); + ASSERT_NE(nullptr, retained_bytes); + EXPECT_LT(retained_bytes->value(), static_cast(row_count * sizeof(int64_t))); +} + +TEST_F(VIcebergMergeSinkTest, TestMatchedRowIdStateAcrossManyFilesAndWrites) { + ObjectPool pool; + MockRuntimeState state; + + DataTypes types {std::make_shared(), + std::make_shared(DataTypes {std::make_shared(), + std::make_shared()}, + Strings {"file_path", "row_position"}), + std::make_shared(), std::make_shared()}; + MockRowDescriptor row_desc(types, &pool); + + auto output_exprs = build_output_exprs(&pool, &state, row_desc); + auto sink = std::make_shared(build_sink(), output_exprs, nullptr, nullptr); + sink->set_skip_io(true); + + ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok()); + RuntimeProfile profile("iceberg_merge_sink"); + ASSERT_TRUE(sink->open(&state, &profile).ok()); + + auto* retained_bytes = profile.get_counter("MatchedRowIdStateBytes"); + ASSERT_NE(nullptr, retained_bytes); + constexpr size_t files_per_write = 32; + constexpr size_t write_count = 64; + std::vector operations(files_per_write, 3); + int64_t previous_bytes = 0; + for (size_t write_index = 0; write_index < write_count; ++write_index) { + Block block = build_block_with_ops(operations, true, write_index * files_per_write); + ASSERT_TRUE(sink->write(&state, block).ok()); + EXPECT_GT(retained_bytes->value(), previous_bytes); + previous_bytes = retained_bytes->value(); + } + + EXPECT_EQ(files_per_write * write_count, sink->_matched_row_positions.size()); + EXPECT_EQ(static_cast(sink->_matched_row_id_state_size), retained_bytes->value()); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp index 94b8aec8c77ea6..974eb817e8872a 100644 --- a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp +++ b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp @@ -520,4 +520,30 @@ TEST_F(PartitionTransformersTest, test_nullable_column_integer_truncate_transfor } } +TEST_F(PartitionTransformersTest, test_nullable_column_string_truncate_transform) { + auto column = ColumnNullable::create(ColumnString::create(), ColumnUInt8::create()); + column->insert_data(nullptr, 0); + column->insert_data("iceberg", sizeof("iceberg") - 1); + column->insert_data("db", sizeof("db") - 1); + ColumnWithTypeAndName test_string( + column->get_ptr(), + std::make_shared(std::make_shared()), "test_string"); + + Block block({test_string}); + auto source_type = DataTypeFactory::instance().create_data_type(TYPE_STRING, true); + StringTruncatePartitionColumnTransform transform(source_type, 3); + + auto result = transform.apply(block, 0); + + const auto* result_column = assert_cast(result.column.get()); + const auto* result_strings = + assert_cast(result_column->get_nested_column_ptr().get()); + EXPECT_EQ(3, result_column->size()); + EXPECT_EQ(Field::create_field(1), result_column->get_null_map_column()[0]); + EXPECT_EQ(Field::create_field(0), result_column->get_null_map_column()[1]); + EXPECT_EQ(Field::create_field(0), result_column->get_null_map_column()[2]); + EXPECT_EQ("ice", result_strings->get_data_at(1).to_string()); + EXPECT_EQ("db", result_strings->get_data_at(2).to_string()); +} + } // namespace doris diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 195c78c62ab72a..57a3662c571447 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2221,7 +2221,7 @@ public class Config extends ConfigBase { * Max data version of backends serialize block. */ @ConfField(mutable = false) - public static int max_be_exec_version = 10; + public static int max_be_exec_version = 11; /** * Min data version of backends serialize block. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index 07f22ef5a54fe1..016831e26aa280 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -197,6 +197,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx if (tableExist(db.getRemoteName(), tableName)) { if (createTableInfo.isIfNotExists()) { LOG.info("create table[{}] which already exists", tableName); + // Existing-table success skips the normal post-create hook, so refresh names here. + resetTableNameCache(dbName); return true; } else { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); @@ -214,6 +216,8 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx if (dorisTable != null) { if (createTableInfo.isIfNotExists()) { LOG.info("create table[{}] which already exists", tableName); + // Every successful no-op bypasses the normal post-create hook and must refresh names. + resetTableNameCache(dbName); return true; } else { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); @@ -225,9 +229,19 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx Schema schema = toPaimonSchema(columns, createTableInfo.getPartitionDesc(), createTableInfo.getProperties()); try { + // Let Paimon report a concurrent winner so callers can distinguish an existing table + // from the table created by this statement before deciding whether rollback is owned. catalog.createTable(new Identifier(createTableInfo.getDbName(), createTableInfo.getTableName()), - schema, createTableInfo.isIfNotExists()); - } catch (TableAlreadyExistException | DatabaseNotExistException e) { + schema, false); + } catch (TableAlreadyExistException e) { + if (createTableInfo.isIfNotExists()) { + LOG.info("create table[{}] which already exists", tableName); + // A concurrent remote creator also bypasses the normal post-create hook. + resetTableNameCache(dbName); + return true; + } + throw new RuntimeException(e); + } catch (DatabaseNotExistException e) { throw new RuntimeException(e); } return false; @@ -278,12 +292,17 @@ private DataType toPaimonType(Type type) { @Override public void afterCreateTable(String dbName, String tblName) { + Optional> db = resetTableNameCache(dbName); + LOG.info("after create table {}.{}.{}, is db exists: {}", + dorisCatalog.getName(), dbName, tblName, db.isPresent()); + } + + private Optional> resetTableNameCache(String dbName) { Optional> db = dorisCatalog.getDbForReplay(dbName); if (db.isPresent()) { db.get().resetMetaCacheNames(); } - LOG.info("after create table {}.{}.{}, is db exists: {}", - dorisCatalog.getName(), dbName, tblName, db.isPresent()); + return db; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 6e1393e1b51624..44ff35ab1d52a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -684,7 +684,8 @@ public PlanFragment visitPhysicalIcebergMergeSink(PhysicalIcebergMergeSink icebergMergeSink, PlanContext context) { - if (connectContext != null && !connectContext.getSessionVariable().enableStrictConsistencyDml) { + if (!icebergMergeSink.isRequireMergeCardinalityCheck() + && connectContext != null + && !connectContext.getSessionVariable().enableStrictConsistencyDml) { addRequestPropertyToChildren(PhysicalProperties.ANY); } else { + // SQL MERGE cardinality is mandatory even when optional UPDATE consistency is disabled. addRequestPropertyToChildren(icebergMergeSink.getRequirePhysicalProperties()); } return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java index 9447aaf09d18c0..85b813b7fd60d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java @@ -39,6 +39,7 @@ public Rule build() { sink.getCols(), sink.getOutputExprs(), sink.getDeleteContext(), + sink.isRequireMergeCardinalityCheck(), Optional.empty(), sink.getLogicalProperties(), null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 13b7e671f7dc24..bdae06f0e6d614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -18,10 +18,12 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.analysis.StmtType; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; @@ -99,6 +101,19 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { LOG.debug("Nereids start to execute the ctas command, query id: {}, tableName: {}", ctx.queryId(), createTableInfo.getTableName()); } + LogicalPlan sinkQuery = null; + if (!createTableInfo.isIfNotExists()) { + // An existence probe is used only to preserve the catalog diagnostic; creation still + // goes through the atomic catalog API and is the sole proof of ownership. + if (targetTableExists(ctx)) { + throw new AnalysisException(ErrorCode.ERR_TABLE_EXISTS_ERROR.formatErrorMsg( + createTableInfo.getTableName())); + } + // Reject unsupported destinations before publishing metadata; rollback by table name + // cannot distinguish this CTAS table from a concurrent replacement with the same name. + sinkQuery = UnboundTableSinkCreator.createUnboundTableSink(createTableInfo.getTableNameParts(), + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), query); + } try { if (Env.getCurrentEnv().createTable(this.createTableInfo)) { return; @@ -107,12 +122,16 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { throw new AnalysisException(e.getMessage(), e.getCause()); } - query = UnboundTableSinkCreator.createUnboundTableSink(createTableInfo.getTableNameParts(), - ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), query); try { + if (sinkQuery == null) { + // IF NOT EXISTS must honor the catalog's atomic existing-table result before sink + // validation, otherwise an unsupported connector turns the required no-op into an error. + sinkQuery = UnboundTableSinkCreator.createUnboundTableSink(createTableInfo.getTableNameParts(), + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), query); + } InsertIntoTableCommand insertCommand = null; if (!FeConstants.runningUnitTest) { - insertCommand = new InsertIntoTableCommand(query, Optional.empty(), + insertCommand = new InsertIntoTableCommand(sinkQuery, Optional.empty(), Optional.empty(), Optional.empty(), true, Optional.empty()); insertCommand.run(ctx, executor); if (ctx.getState().getStateType() == MysqlStateType.OK) { @@ -226,6 +245,16 @@ void handleFallbackFailedCtas(ConnectContext ctx) { } } + boolean targetTableExists(ConnectContext ctx) { + List qualifiedName = RelationUtil.getQualifierName(ctx, createTableInfo.getTableNameParts()); + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(qualifiedName.get(0)); + if (catalog == null) { + return false; + } + DatabaseIf database = catalog.getDbNullable(qualifiedName.get(1)); + return database != null && database.isTableExist(qualifiedName.get(2)); + } + private String getAutoRangePartitionNameOrNull() { try { if (createTableInfo.getPartitionTableInfo().isAutoPartition() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java index 59eb6a6d6acc26..4d4b9b2014fec6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java @@ -465,6 +465,7 @@ private LogicalPlan buildMergePlan(ConnectContext ctx, IcebergExternalTable iceb icebergTable.getBaseSchema(true), outputExprs, deleteCtx, + true, Optional.empty(), Optional.empty(), projectPlan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java index 8759343e055565..d9d0ab51b58a78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java @@ -241,6 +241,7 @@ private LogicalPlan buildMergePlan(ConnectContext ctx, LogicalPlan logicalQuery, icebergTable.getBaseSchema(true), outputExprs, deleteCtx, + false, Optional.empty(), Optional.empty(), queryPlan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java index 7f528020890dde..c8198c27dcce22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java @@ -47,6 +47,7 @@ public class LogicalIcebergMergeSink extends LogicalTab private final IcebergExternalDatabase database; private final IcebergExternalTable targetTable; private final DeleteCommandContext deleteContext; + private final boolean requireMergeCardinalityCheck; /** * Constructor @@ -56,6 +57,7 @@ public LogicalIcebergMergeSink(IcebergExternalDatabase database, List cols, List outputExprs, DeleteCommandContext deleteContext, + boolean requireMergeCardinalityCheck, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { @@ -63,6 +65,7 @@ public LogicalIcebergMergeSink(IcebergExternalDatabase database, this.database = Objects.requireNonNull(database, "database != null in LogicalIcebergMergeSink"); this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalIcebergMergeSink"); this.deleteContext = Objects.requireNonNull(deleteContext, "deleteContext != null in LogicalIcebergMergeSink"); + this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; } public Plan withChildAndUpdateOutput(Plan child) { @@ -70,19 +73,19 @@ public Plan withChildAndUpdateOutput(Plan child) { .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); return new LogicalIcebergMergeSink<>(database, targetTable, cols, output, - deleteContext, Optional.empty(), Optional.empty(), child); + deleteContext, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), child); } @Override public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "LogicalIcebergMergeSink only accepts one child"); return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, - deleteContext, Optional.empty(), Optional.empty(), children.get(0)); + deleteContext, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), children.get(0)); } public LogicalIcebergMergeSink withOutputExprs(List outputExprs) { return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, - deleteContext, Optional.empty(), Optional.empty(), child()); + deleteContext, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), child()); } public IcebergExternalDatabase getDatabase() { @@ -97,6 +100,10 @@ public DeleteCommandContext getDeleteContext() { return deleteContext; } + public boolean isRequireMergeCardinalityCheck() { + return requireMergeCardinalityCheck; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -112,12 +119,14 @@ public boolean equals(Object o) { return Objects.equals(database, that.database) && Objects.equals(targetTable, that.targetTable) && Objects.equals(deleteContext, that.deleteContext) + && requireMergeCardinalityCheck == that.requireMergeCardinalityCheck && Objects.equals(cols, that.cols); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), database, targetTable, cols, deleteContext); + return Objects.hash(super.hashCode(), database, targetTable, cols, deleteContext, + requireMergeCardinalityCheck); } @Override @@ -127,7 +136,8 @@ public String toString() { "database", database.getFullName(), "targetTable", targetTable.getName(), "cols", cols, - "deleteFileType", deleteContext.getDeleteFileType()); + "deleteFileType", deleteContext.getDeleteFileType(), + "requireMergeCardinalityCheck", requireMergeCardinalityCheck); } @Override @@ -138,13 +148,15 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, - deleteContext, groupExpression, Optional.of(getLogicalProperties()), child()); + deleteContext, requireMergeCardinalityCheck, + groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, - deleteContext, groupExpression, logicalProperties, children.get(0)); + deleteContext, requireMergeCardinalityCheck, + groupExpression, logicalProperties, children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java index 0281ad23243496..e182c5adc06302 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java @@ -22,7 +22,6 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMergeOperation; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.properties.DistributionSpecMerge; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -56,6 +55,7 @@ */ public class PhysicalIcebergMergeSink extends PhysicalBaseExternalTableSink { private final DeleteCommandContext deleteContext; + private final boolean requireMergeCardinalityCheck; /** * Constructor @@ -65,10 +65,12 @@ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, List cols, List outputExprs, DeleteCommandContext deleteContext, + boolean requireMergeCardinalityCheck, Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(database, targetTable, cols, outputExprs, deleteContext, groupExpression, logicalProperties, + this(database, targetTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, + groupExpression, logicalProperties, PhysicalProperties.GATHER, null, child); } @@ -80,6 +82,7 @@ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, List cols, List outputExprs, DeleteCommandContext deleteContext, + boolean requireMergeCardinalityCheck, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, @@ -89,17 +92,22 @@ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, logicalProperties, physicalProperties, statistics, child); this.deleteContext = Objects.requireNonNull( deleteContext, "deleteContext != null in PhysicalIcebergMergeSink"); + this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; } public DeleteCommandContext getDeleteContext() { return deleteContext; } + public boolean isRequireMergeCardinalityCheck() { + return requireMergeCardinalityCheck; + } + @Override public Plan withChildren(List children) { return new PhysicalIcebergMergeSink<>( (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, - cols, outputExprs, deleteContext, groupExpression, + cols, outputExprs, deleteContext, requireMergeCardinalityCheck, groupExpression, getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @@ -112,7 +120,8 @@ public R accept(PlanVisitor visitor, C context) { public Plan withGroupExpression(Optional groupExpression) { return new PhysicalIcebergMergeSink<>( (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, groupExpression, getLogicalProperties(), child()); + deleteContext, requireMergeCardinalityCheck, + groupExpression, getLogicalProperties(), child()); } @Override @@ -120,14 +129,16 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new PhysicalIcebergMergeSink<>( (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, groupExpression, logicalProperties.get(), children.get(0)); + deleteContext, requireMergeCardinalityCheck, + groupExpression, logicalProperties.get(), children.get(0)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalIcebergMergeSink<>( (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); + deleteContext, requireMergeCardinalityCheck, + groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); } @Override @@ -142,12 +153,13 @@ public boolean equals(Object o) { return false; } PhysicalIcebergMergeSink that = (PhysicalIcebergMergeSink) o; - return Objects.equals(deleteContext, that.deleteContext); + return Objects.equals(deleteContext, that.deleteContext) + && requireMergeCardinalityCheck == that.requireMergeCardinalityCheck; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), deleteContext); + return Objects.hash(super.hashCode(), deleteContext, requireMergeCardinalityCheck); } /** @@ -172,8 +184,16 @@ public PhysicalProperties getRequirePhysicalProperties() { ConnectContext ctx = ConnectContext.get(); if (ctx == null || !ctx.getSessionVariable().isEnableIcebergMergePartitioning()) { - if (rowIdExprId != null) { - return PhysicalProperties.createHash(ImmutableList.of(rowIdExprId), ShuffleType.REQUIRE); + if (rowIdExprId != null && operationExprId != null) { + // Route only delete images by row ID; unmatched inserts have NULL row IDs and must + // remain distributable instead of collapsing onto one exchange channel. + return new PhysicalProperties(new DistributionSpecMerge( + operationExprId, + ImmutableList.of(), + ImmutableList.of(rowIdExprId), + true, + ImmutableList.of(), + null)); } return PhysicalProperties.GATHER; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java index 9775f81c172fae..c57ea76c75dd1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java @@ -64,6 +64,7 @@ public class IcebergMergeSink extends BaseExternalTableDataSink { private final IcebergExternalTable targetTable; private final DeleteCommandContext deleteContext; + private final boolean requireMergeCardinalityCheck; private List rewritableDeleteFileSets = Collections.emptyList(); private static final HashSet supportedTypes = new HashSet() {{ @@ -74,13 +75,15 @@ public class IcebergMergeSink extends BaseExternalTableDataSink { // Store PropertiesMap, including vended credentials or static credentials private Map storagePropertiesMap; - public IcebergMergeSink(IcebergExternalTable targetTable, DeleteCommandContext deleteContext) { + public IcebergMergeSink(IcebergExternalTable targetTable, DeleteCommandContext deleteContext, + boolean requireMergeCardinalityCheck) { super(); if (targetTable.isView()) { throw new UnsupportedOperationException("UPDATE on iceberg view is not supported"); } this.targetTable = targetTable; this.deleteContext = deleteContext; + this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; IcebergExternalCatalog catalog = (IcebergExternalCatalog) targetTable.getCatalog(); storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( @@ -132,6 +135,8 @@ public void bindDataSink(Optional insertCtx) tSink.setFormatVersion(formatVersion); tSink.setSchemaJson(SchemaParser.toJson(schema)); tSink.setCollectColumnStats(IcebergUtils.shouldCollectColumnStats(icebergTable, schema)); + // UPDATE and SQL MERGE share this sink, but only SQL MERGE has the one-source-row invariant. + tSink.setRequireMergeCardinalityCheck(requireMergeCardinalityCheck); // partition spec if (icebergTable.spec().isPartitioned()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index b4a1c6a5d1b235..556ff50225f8ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.glue.LogicalPlanAdapter; -import org.apache.doris.nereids.properties.DistributionSpecHash; import org.apache.doris.nereids.properties.DistributionSpecMerge; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Alias; @@ -462,6 +461,68 @@ public void testIcebergMergeIntoExchangeUsesMergePartitioningWhenEnabled() throw } } + @Test + public void testIcebergMergeIntoExchangeIgnoresStrictConsistencySwitch() throws Exception { + useIceberg(); + boolean previousMergePartitioning = connectContext.getSessionVariable().enableIcebergMergePartitioning; + boolean previousStrictConsistency = connectContext.getSessionVariable().enableStrictConsistencyDml; + connectContext.getSessionVariable().enableIcebergMergePartitioning = true; + connectContext.getSessionVariable().enableStrictConsistencyDml = false; + try { + String sql = "merge into " + tableName + " t " + + "using (select 1 as id, 'name1' as name, 10 as age, 1 as score, 1.23 as amount) s " + + "on t.id = s.id " + + "when matched then update set name = s.name"; + LogicalPlan mergePlan = parseStmt(sql); + Plan explainPlan = ((MergeIntoCommand) mergePlan).getExplainPlan(connectContext); + PhysicalPlan physicalPlan = + planPhysicalPlan((LogicalPlan) explainPlan, PhysicalProperties.GATHER, sql); + + PhysicalIcebergMergeSink sink = + getSinglePhysicalSink(physicalPlan, PhysicalIcebergMergeSink.class); + Assertions.assertTrue(sink.isRequireMergeCardinalityCheck()); + Assertions.assertTrue(sink.child() instanceof PhysicalDistribute, + "MERGE cardinality requires row-id routing even when strict consistency is disabled\n" + + physicalPlan.treeString()); + Assertions.assertTrue( + ((PhysicalDistribute) sink.child()).getDistributionSpec() instanceof DistributionSpecMerge, + "Missing merge distribution spec\n" + physicalPlan.treeString()); + } finally { + connectContext.getSessionVariable().enableIcebergMergePartitioning = previousMergePartitioning; + connectContext.getSessionVariable().enableStrictConsistencyDml = previousStrictConsistency; + } + } + + @Test + public void testIcebergUpdateAvoidsExchangeWhenStrictConsistencyDisabled() throws Exception { + useIceberg(); + boolean previousMergePartitioning = connectContext.getSessionVariable().enableIcebergMergePartitioning; + boolean previousStrictConsistency = connectContext.getSessionVariable().enableStrictConsistencyDml; + connectContext.getSessionVariable().enableIcebergMergePartitioning = true; + connectContext.getSessionVariable().enableStrictConsistencyDml = false; + try { + // UPDATE FROM may produce the same target row more than once, but SQL MERGE's + // one-source-row cardinality rule must not be applied to UPDATE statements. + String sql = "update " + tableName + " t set name = s.name " + + "from (select 1 as id, 'first' as name union all " + + "select 1 as id, 'second' as name) s where t.id = s.id"; + LogicalPlan updatePlan = parseStmt(sql); + Plan explainPlan = ((UpdateCommand) updatePlan).getExplainPlan(connectContext); + PhysicalPlan physicalPlan = + planPhysicalPlan((LogicalPlan) explainPlan, PhysicalProperties.GATHER, sql); + + PhysicalIcebergMergeSink sink = + getSinglePhysicalSink(physicalPlan, PhysicalIcebergMergeSink.class); + Assertions.assertFalse(sink.isRequireMergeCardinalityCheck()); + Assertions.assertFalse(sink.child() instanceof PhysicalDistribute, + "Strict-consistency-off UPDATE should not add an Iceberg merge exchange\n" + + physicalPlan.treeString()); + } finally { + connectContext.getSessionVariable().enableIcebergMergePartitioning = previousMergePartitioning; + connectContext.getSessionVariable().enableStrictConsistencyDml = previousStrictConsistency; + } + } + @Test public void testIcebergUpdateCastsConstantForSmallintAndDecimal() throws Exception { useIceberg(); @@ -477,7 +538,7 @@ public void testIcebergUpdateCastsConstantForSmallintAndDecimal() throws Excepti } @Test - public void testIcebergUpdateExchangeUsesRowIdOnlyWhenDisabled() throws Exception { + public void testIcebergUpdateExchangeRoutesOnlyDeletesByRowIdWhenDisabled() throws Exception { useIceberg(); boolean previous = connectContext.getSessionVariable().enableIcebergMergePartitioning; connectContext.getSessionVariable().enableIcebergMergePartitioning = false; @@ -490,19 +551,93 @@ public void testIcebergUpdateExchangeUsesRowIdOnlyWhenDisabled() throws Exceptio PhysicalIcebergMergeSink sink = getSinglePhysicalSink(physicalPlan, PhysicalIcebergMergeSink.class); + ExprId operationExprId = findOperationExprId(sink.child().getOutput()); ExprId rowIdExprId = findRowIdExprId(sink.child().getOutput()); Assertions.assertTrue(sink.child() instanceof PhysicalDistribute, - "Missing row_id exchange\n" + physicalPlan.treeString()); + "Missing operation-aware exchange\n" + physicalPlan.treeString()); PhysicalDistribute distribute = (PhysicalDistribute) sink.child(); - Assertions.assertTrue(distribute.getDistributionSpec() instanceof DistributionSpecHash, - "Missing row_id hash distribution\n" + physicalPlan.treeString()); - DistributionSpecHash hash = (DistributionSpecHash) distribute.getDistributionSpec(); - Assertions.assertEquals(ImmutableList.of(rowIdExprId), hash.getOrderedShuffledColumns()); + Assertions.assertTrue(distribute.getDistributionSpec() instanceof DistributionSpecMerge, + "Missing operation-aware merge distribution\n" + physicalPlan.treeString()); + DistributionSpecMerge spec = (DistributionSpecMerge) distribute.getDistributionSpec(); + Assertions.assertEquals(operationExprId, spec.getOperationExprId()); + Assertions.assertTrue(spec.isInsertRandom()); + Assertions.assertEquals(ImmutableList.of(rowIdExprId), spec.getDeletePartitionExprIds()); } finally { connectContext.getSessionVariable().enableIcebergMergePartitioning = previous; } } + @Test + public void testIcebergInsertOnlyMergeDoesNotHashNullRowIdsWhenStrictConsistencyDisabled() + throws Exception { + useIceberg(); + boolean previousMergePartitioning = connectContext.getSessionVariable().enableIcebergMergePartitioning; + boolean previousStrictConsistency = connectContext.getSessionVariable().enableStrictConsistencyDml; + connectContext.getSessionVariable().enableIcebergMergePartitioning = false; + connectContext.getSessionVariable().enableStrictConsistencyDml = false; + try { + String sql = "merge into " + tableName + " t " + + "using (select 1 as id, 'name1' as name, 10 as age, 1 as score, 1.23 as amount) s " + + "on t.id = s.id " + + "when not matched then insert (id, name, age, score, amount) " + + "values (s.id, s.name, s.age, s.score, s.amount)"; + LogicalPlan mergePlan = parseStmt(sql); + Plan explainPlan = ((MergeIntoCommand) mergePlan).getExplainPlan(connectContext); + PhysicalPlan physicalPlan = + planPhysicalPlan((LogicalPlan) explainPlan, PhysicalProperties.GATHER, sql); + + PhysicalIcebergMergeSink sink = + getSinglePhysicalSink(physicalPlan, PhysicalIcebergMergeSink.class); + Assertions.assertTrue(sink.child() instanceof PhysicalDistribute, + "Missing operation-aware exchange\n" + physicalPlan.treeString()); + DistributionSpecMerge spec = (DistributionSpecMerge) + ((PhysicalDistribute) sink.child()).getDistributionSpec(); + Assertions.assertTrue(spec.isInsertRandom()); + Assertions.assertTrue(spec.getInsertPartitionExprIds().isEmpty()); + Assertions.assertEquals( + ImmutableList.of(findRowIdExprId(sink.child().getOutput())), + spec.getDeletePartitionExprIds()); + } finally { + connectContext.getSessionVariable().enableIcebergMergePartitioning = previousMergePartitioning; + connectContext.getSessionVariable().enableStrictConsistencyDml = previousStrictConsistency; + } + } + + @Test + public void testIcebergMixedMergeRoutesInsertsRandomlyWhenStrictConsistencyDisabled() + throws Exception { + useIceberg(); + boolean previousMergePartitioning = connectContext.getSessionVariable().enableIcebergMergePartitioning; + boolean previousStrictConsistency = connectContext.getSessionVariable().enableStrictConsistencyDml; + connectContext.getSessionVariable().enableIcebergMergePartitioning = false; + connectContext.getSessionVariable().enableStrictConsistencyDml = false; + try { + String sql = "merge into " + tableName + " t " + + "using (select 1 as id, 'name1' as name, 10 as age, 1 as score, 1.23 as amount) s " + + "on t.id = s.id " + + "when matched then update set name = s.name " + + "when not matched then insert (id, name, age, score, amount) " + + "values (s.id, s.name, s.age, s.score, s.amount)"; + LogicalPlan mergePlan = parseStmt(sql); + Plan explainPlan = ((MergeIntoCommand) mergePlan).getExplainPlan(connectContext); + PhysicalPlan physicalPlan = + planPhysicalPlan((LogicalPlan) explainPlan, PhysicalProperties.GATHER, sql); + + PhysicalIcebergMergeSink sink = + getSinglePhysicalSink(physicalPlan, PhysicalIcebergMergeSink.class); + DistributionSpecMerge spec = (DistributionSpecMerge) + ((PhysicalDistribute) sink.child()).getDistributionSpec(); + Assertions.assertTrue(spec.isInsertRandom()); + Assertions.assertTrue(spec.getInsertPartitionExprIds().isEmpty()); + Assertions.assertEquals( + ImmutableList.of(findRowIdExprId(sink.child().getOutput())), + spec.getDeletePartitionExprIds()); + } finally { + connectContext.getSessionVariable().enableIcebergMergePartitioning = previousMergePartitioning; + connectContext.getSessionVariable().enableStrictConsistencyDml = previousStrictConsistency; + } + } + @Test public void testIcebergUpdateExchangeUsesMergePartitioningWhenEnabled() throws Exception { useIceberg(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java index 61fd4d83063a49..3135df2541f4d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java @@ -22,6 +22,8 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogFactory; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -34,6 +36,7 @@ import org.apache.paimon.catalog.FileSystemCatalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.hive.HiveCatalog; +import org.apache.paimon.schema.Schema; import org.apache.paimon.table.Table; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; @@ -48,12 +51,14 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.jupiter.api.Assertions; +import org.mockito.Mockito; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -263,13 +268,70 @@ public void testModifyColumnPreservesRemoteTypeBehindLossyProjection() Assert.assertEquals(projectedColumn.isAllowNull(), requestedType.isNullable()); } + @Test + public void testIfNotExistsRefreshesNamesWhenRemoteTableExists() throws Exception { + String tableName = getTableName(); + Catalog remoteCatalog = Mockito.mock(Catalog.class); + ExternalCatalog dorisCatalog = Mockito.mock(ExternalCatalog.class); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(database).when(dorisCatalog).getDbNullable(dbName); + Mockito.doReturn(Optional.of(database)).when(dorisCatalog).getDbForReplay(dbName); + Mockito.when(database.getRemoteName()).thenReturn(dbName); + + PaimonMetadataOps existingTableOps = new PaimonMetadataOps(dorisCatalog, remoteCatalog) { + @Override + public boolean tableExist(String ignoredDbName, String ignoredTableName) { + return true; + } + }; + CreateTableInfo createTableInfo = parseCreateTableInfo( + "create table if not exists " + dbName + "." + tableName + " (id int) engine = paimon"); + + Assert.assertTrue(existingTableOps.performCreateTable(createTableInfo)); + Mockito.verify(database).resetMetaCacheNames(); + Mockito.verify(remoteCatalog, Mockito.never()).createTable( + Mockito.any(Identifier.class), Mockito.any(Schema.class), Mockito.anyBoolean()); + } + + @Test + public void testIfNotExistsReportsConcurrentWinner() throws Exception { + String tableName = getTableName(); + Identifier identifier = new Identifier(dbName, tableName); + Catalog remoteCatalog = Mockito.mock(Catalog.class); + ExternalCatalog dorisCatalog = Mockito.mock(ExternalCatalog.class); + ExternalDatabase database = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(database).when(dorisCatalog).getDbNullable(dbName); + Mockito.doReturn(Optional.of(database)).when(dorisCatalog).getDbForReplay(dbName); + Mockito.when(database.getRemoteName()).thenReturn(dbName); + Mockito.when(database.getTableNullable(tableName)).thenReturn(null); + + PaimonMetadataOps raceOps = new PaimonMetadataOps(dorisCatalog, remoteCatalog) { + @Override + public boolean tableExist(String ignoredDbName, String ignoredTableName) { + return false; + } + }; + CreateTableInfo createTableInfo = parseCreateTableInfo( + "create table if not exists " + dbName + "." + tableName + " (id int) engine = paimon"); + Mockito.doThrow(new Catalog.TableAlreadyExistException(identifier)) + .when(remoteCatalog).createTable( + Mockito.eq(identifier), Mockito.any(Schema.class), Mockito.eq(false)); + + Assert.assertTrue(raceOps.performCreateTable(createTableInfo)); + Mockito.verify(database).resetMetaCacheNames(); + } + public void createTable(String sql) throws UserException { + ops.createTable(parseCreateTableInfo(sql)); + } + + private CreateTableInfo parseCreateTableInfo(String sql) throws UserException { LogicalPlan plan = new NereidsParser().parseSingle(sql); Assertions.assertTrue(plan instanceof CreateTableCommand); CreateTableInfo createTableInfo = ((CreateTableCommand) plan).getCreateTableInfo(); createTableInfo.setIsExternal(true); createTableInfo.analyzeEngine(); - ops.createTable(createTableInfo); + return createTableInfo; } public String getTableName() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java new file mode 100644 index 00000000000000..38324e828aede5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; +import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.List; +import java.util.Optional; + +class CreateTableCommandTest { + + @Test + void ctasIfNotExistsReturnsBeforeUnsupportedSinkValidation() throws Exception { + LogicalPlan query = Mockito.mock(LogicalPlan.class); + CreateTableInfo createTableInfo = Mockito.mock(CreateTableInfo.class); + ConnectContext context = Mockito.mock(ConnectContext.class); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + Env env = Mockito.mock(Env.class); + List tableNameParts = ImmutableList.of("catalog", "database", "target"); + + Mockito.when(createTableInfo.getTableNameParts()).thenReturn(tableNameParts); + Mockito.when(createTableInfo.isIfNotExists()).thenReturn(true); + Mockito.when(env.createTable(createTableInfo)).thenReturn(true); + CreateTableCommand command = Mockito.spy( + new CreateTableCommand(Optional.of(query), createTableInfo)); + Mockito.doNothing().when(command).validateCreateTableAsSelect(context, query); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class); + MockedStatic sinkMock = + Mockito.mockStatic(UnboundTableSinkCreator.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(env); + + org.junit.jupiter.api.Assertions.assertDoesNotThrow( + () -> command.run(context, executor)); + + sinkMock.verifyNoInteractions(); + Mockito.verify(env).createTable(createTableInfo); + } + } + + @Test + void ctasValidatesSinkBeforePublishingMetadata() throws Exception { + LogicalPlan query = Mockito.mock(LogicalPlan.class); + CreateTableInfo createTableInfo = Mockito.mock(CreateTableInfo.class); + ConnectContext context = Mockito.mock(ConnectContext.class); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + Env env = Mockito.mock(Env.class); + List tableNameParts = ImmutableList.of("catalog", "database", "target"); + + Mockito.when(createTableInfo.getTableNameParts()).thenReturn(tableNameParts); + CreateTableCommand command = Mockito.spy( + new CreateTableCommand(Optional.of(query), createTableInfo)); + Mockito.doNothing().when(command).validateCreateTableAsSelect(context, query); + Mockito.doReturn(false).when(command).targetTableExists(context); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class); + MockedStatic sinkMock = + Mockito.mockStatic(UnboundTableSinkCreator.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(env); + sinkMock.when(() -> UnboundTableSinkCreator.createUnboundTableSink( + tableNameParts, ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), query)) + .thenThrow(new UserException("unsupported sink")); + + org.junit.jupiter.api.Assertions.assertThrows( + Exception.class, () -> command.run(context, executor)); + + Mockito.verify(env, Mockito.never()).createTable(createTableInfo); + Mockito.verify(env, Mockito.never()).dropTable( + Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), + Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); + } + } + + @Test + void existingCtasTargetPrecedesUnsupportedSinkValidation() throws Exception { + LogicalPlan query = Mockito.mock(LogicalPlan.class); + CreateTableInfo createTableInfo = Mockito.mock(CreateTableInfo.class); + ConnectContext context = Mockito.mock(ConnectContext.class); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + Env env = Mockito.mock(Env.class); + List tableNameParts = ImmutableList.of("catalog", "database", "target"); + + Mockito.when(createTableInfo.getTableNameParts()).thenReturn(tableNameParts); + Mockito.when(createTableInfo.getTableName()).thenReturn("target"); + CreateTableCommand command = Mockito.spy( + new CreateTableCommand(Optional.of(query), createTableInfo)); + Mockito.doNothing().when(command).validateCreateTableAsSelect(context, query); + Mockito.doReturn(true).when(command).targetTableExists(context); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class); + MockedStatic sinkMock = + Mockito.mockStatic(UnboundTableSinkCreator.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(env); + Exception exception = org.junit.jupiter.api.Assertions.assertThrows( + Exception.class, () -> command.run(context, executor)); + + org.junit.jupiter.api.Assertions.assertTrue(exception.getMessage().contains("already exists")); + sinkMock.verifyNoInteractions(); + Mockito.verify(env, Mockito.never()).createTable(createTableInfo); + Mockito.verify(env, Mockito.never()).dropTable( + Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), + Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java index 8015cf5feb3bb3..7ea40b11739835 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java @@ -99,7 +99,8 @@ public void testFinalizeSinkAndBeforeExecPropagateRewritableDeleteMetadata() thr ctx.setThreadLocalInfo(); IcebergMergeExecutor executor = new IcebergMergeExecutor(ctx, table, "label", planner, false, -1L); - IcebergMergeSink sink = new IcebergMergeSink(table, new org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext()); + IcebergMergeSink sink = new IcebergMergeSink(table, + new org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext(), true); executor.finalizeSinkForMerge(null, sink, null); TDataSink tDataSink = getTDataSink(sink); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java index 74d231d3154a25..46eb5e0324333d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java @@ -47,7 +47,8 @@ public class IcebergMergeSinkTest { @Test public void testBindDataSinkIncludesRowLineageSchemaAndRewritableDeleteFileSetsForV3() throws Exception { - IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(3), new DeleteCommandContext()); + IcebergMergeSink sink = new IcebergMergeSink( + mockIcebergExternalTable(3), new DeleteCommandContext(), true); sink.setRewritableDeleteFileSets(Collections.singletonList(buildDeleteFileSet())); sink.bindDataSink(Optional.empty()); @@ -58,11 +59,14 @@ public void testBindDataSinkIncludesRowLineageSchemaAndRewritableDeleteFileSetsF Assertions.assertTrue(thriftSink.getSchemaJson().contains( IcebergUtils.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL)); Assertions.assertEquals(1, thriftSink.getRewritableDeleteFileSetsSize()); + Assertions.assertTrue(thriftSink.isSetRequireMergeCardinalityCheck()); + Assertions.assertTrue(thriftSink.isRequireMergeCardinalityCheck()); } @Test public void testBindDataSinkSkipsRewritableDeleteFileSetsAndRowLineageSchemaForV2() throws Exception { - IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2), new DeleteCommandContext()); + IcebergMergeSink sink = new IcebergMergeSink( + mockIcebergExternalTable(2), new DeleteCommandContext(), false); sink.setRewritableDeleteFileSets(Collections.singletonList(buildDeleteFileSet())); sink.bindDataSink(Optional.empty()); @@ -73,12 +77,14 @@ public void testBindDataSinkSkipsRewritableDeleteFileSetsAndRowLineageSchemaForV Assertions.assertFalse(thriftSink.getSchemaJson().contains(IcebergUtils.ICEBERG_ROW_ID_COL)); Assertions.assertFalse(thriftSink.getSchemaJson().contains( IcebergUtils.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL)); + Assertions.assertTrue(thriftSink.isSetRequireMergeCardinalityCheck()); + Assertions.assertFalse(thriftSink.isRequireMergeCardinalityCheck()); } @Test public void testBindDataSinkDisablesColumnStatsWhenAllMetricsAreNone() throws Exception { IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, ImmutableMap.of( - TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")), new DeleteCommandContext()); + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")), new DeleteCommandContext(), false); sink.bindDataSink(Optional.empty()); @@ -92,7 +98,7 @@ public void testBindDataSinkKeepsColumnStatsForMetricsOverride() throws Exceptio IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, ImmutableMap.of( TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "counts")), - new DeleteCommandContext()); + new DeleteCommandContext(), false); sink.bindDataSink(Optional.empty()); @@ -106,7 +112,7 @@ public void testBindDataSinkKeepsColumnStatsForV3LineageFields() throws Exceptio IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(3, ImmutableMap.of( TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts", TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "none")), - new DeleteCommandContext()); + new DeleteCommandContext(), false); sink.bindDataSink(Optional.empty()); @@ -123,7 +129,7 @@ public void testBindDataSinkKeepsColumnStatsForOrcTopLevelComplexField() throws TableProperties.DEFAULT_FILE_FORMAT, "orc", TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "items", "counts")), - new DeleteCommandContext()); + new DeleteCommandContext(), false); sink.bindDataSink(Optional.empty()); diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 47ad91e3f274ff..942fd9962ba580 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -537,6 +537,8 @@ struct TIcebergMergeSink { 13: optional list broker_addresses; // Unset keeps collection enabled for rolling upgrades with older FEs. 14: optional bool collect_column_stats; + // Unset preserves old-FE UPDATE behavior; execution version gates SQL MERGE validation. + 15: optional bool require_merge_cardinality_check; // delete side (position delete only) 20: optional TFileContent delete_type diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out index 3f8c510addc24d..5d765df5caca66 100644 --- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out @@ -1,3 +1,5 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !duplicate_source_atomic_state -- 1 A committed +-- !sibling_close_atomic_state -- +1 A committed diff --git a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md index f3760f4db68ced..5a551b6d3d8e34 100644 --- a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md +++ b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md @@ -43,8 +43,9 @@ boundaries. Paimon P0 was incomplete. No suite selected a `merge-engine`, and write rejection was not checked across all DML shapes with pre/post data and snapshot invariants. PM01-PM03 below close those gaps. -PM04 exposes one remaining product defect as an opt-in negative regression: failed Paimon CTAS leaves -target metadata. Streaming writes, overwrite, delete/update and merge listed as unsupported by the +PM04 now runs as an active negative regression: failed Paimon CTAS is rejected without leaving target +metadata, while `IF NOT EXISTS` remains a no-op for an existing table. Streaming writes, overwrite, +delete/update and merge listed as unsupported by the Paimon ecosystem matrix are boundary tests rather than positive Doris P0 contracts. ## Risks @@ -75,7 +76,7 @@ Paimon ecosystem matrix are boundary tests rather than positive Doris P0 contrac | Paimon | Deletion vectors, upsert/delete visibility and data/system tables | Covered | `test_paimon_deletion_vector`, `paimon_data_system_table`, `paimon_system_table` | | Paimon | Catalog/database/table create and drop | Covered | `test_create_paimon_table` | | Paimon | Doris data write-back | Negative boundary covered | `test_paimon_write_boundary` | -| Paimon | Failed CTAS metadata atomicity | Isolated known-bug regression | `test_paimon_ctas_atomicity_negative` | +| Paimon | Failed CTAS metadata atomicity | Active negative regression | `test_paimon_ctas_atomicity_negative` | | Iceberg | V1/V2/V3, Parquet/ORC, position/equality deletes and deletion vectors | Covered | `test_iceberg_position_delete`, `test_iceberg_equality_delete`, `test_iceberg_deletion_vector` | | Iceberg | Schema, partition and sort-order evolution | Covered | `test_iceberg_schema_time_travel_matrix`, `test_iceberg_partition_evolution_format_scanner`, `iceberg_schema_change_ddl` | | Iceberg | Snapshot/timestamp/tag/branch reads and reference actions | Covered | `test_iceberg_time_travel`, `iceberg_query_tag_branch`, `test_iceberg_schema_ref_actions_matrix` | @@ -95,7 +96,7 @@ Detailed schema/time-travel and Iceberg write combinations are maintained in | PM01 | Distinguish all four primary-key merge engines | R01, R02, R04 | Functional, correctness, compatibility | Paimon Parquet/ORC tables | Duplicate keys across several commits | Each engine returns its documented merged row under automatic and forced-JNI routing | | PM02 | Validate dynamic-bucket cross-partition deduplication | R03, R04 | Correctness | Primary key excludes partition key, bucket=-1 | Move one key between partitions | Exactly one current row remains in the new partition | | PM03 | Preserve the Paimon read-only boundary | R05, R10 | Negative, atomicity | Existing Paimon PK table | VALUES, SELECT, OVERWRITE, UPDATE, DELETE, MERGE | Every statement fails before a snapshot or data change | -| PM04 | Reject or roll back Paimon CTAS atomically | R05, R10 | Isolated negative, atomicity | Paimon catalog with no target table | CREATE TABLE AS SELECT | Desired contract: the command fails and no target table remains; current bug leaves the table | +| PM04 | Reject or roll back Paimon CTAS atomically | R05, R10 | Negative, atomicity | Missing and existing Paimon target tables | CREATE TABLE AS SELECT, including IF NOT EXISTS | Unsupported CTAS leaves no target; an existing target makes IF NOT EXISTS a successful no-op | Every P0 risk maps to at least one deterministic positive, boundary, or isolated known-bug regression. Catalog authentication and cloud storage permutations stay in their existing connector diff --git a/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md index 7f0ba5b849c725..6829fd1d4c4fa6 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md +++ b/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md @@ -70,10 +70,10 @@ under the License. | Doris 源 bucket | HASH 固定 bucket、RANDOM bucket、HASH AUTO bucket | 已覆盖（验证通过） | `test_iceberg_write_source_models` | | 分区源类型 | STRING/INT/BIGINT/DATE/DATETIME/DECIMAL 的 bucket 与适用 transform；BOOLEAN identity 与非法 bucket | 已覆盖（验证通过） | `test_iceberg_write_partition_types_null` | | NULL 分区 | identity NULL、数值/decimal bucket 与 truncate NULL、time transform NULL、多列组合 NULL | 已覆盖（验证通过） | `test_iceberg_write_partition_types_null` | -| nullable STRING truncate | nullable STRING 经过 truncate transform 的 INSERT，以及 UPDATE 产生的 Nullable projection 写入 | 已覆盖（隔离负向） | `test_iceberg_write_nullable_truncate_negative` | +| nullable STRING truncate | nullable STRING 经过 truncate transform 的 INSERT，以及 UPDATE 产生的 Nullable projection 写入 | 已覆盖（常规回归） | `test_iceberg_write_nullable_truncate_negative` | | MERGE 完整语义 | 条件 MATCHED、DELETE/UPDATE、多个条件 NOT MATCHED、NULL-safe 与普通 NULL key | 已覆盖（验证通过） | `test_iceberg_write_merge_semantics` | -| MERGE 基数约束 | 多个源行匹配同一目标行必须整句失败且不发布快照 | 已覆盖（隔离负向） | `test_iceberg_write_merge_duplicate_source_negative` | -| MERGE + STRING truncate | required truncate 源列经 MERGE nullable projection 写入 | 已覆盖（隔离负向） | `test_iceberg_write_merge_truncate_negative` | +| MERGE 基数约束 | 多个源行匹配同一目标行必须整句失败且不发布快照 | 已覆盖（常规回归） | `test_iceberg_write_merge_duplicate_source_negative` | +| MERGE + STRING truncate | required truncate 源列经 MERGE nullable projection 写入 | 已覆盖（常规回归） | `test_iceberg_write_merge_truncate_negative` | | branch/tag 写入边界 | branch INSERT/OVERWRITE 隔离；tag 写入和 branch DELETE/UPDATE/MERGE 明确拒绝 | 已覆盖（验证通过） | `test_iceberg_write_branch_dml_boundary` | | nullable 数据 | 顶层 NULL、ARRAY NULL 元素、MAP NULL value、STRUCT NULL child | 已覆盖并增强 | `test_iceberg_write_insert`、`test_iceberg_write_complex_evolution` | | required 列正向与 schema change | required 列合法写入、nullable 列写 NULL、增加 required 列与 nullable→required 拒绝 | 已覆盖（验证通过） | `test_iceberg_write_nullability_atomicity` | @@ -100,9 +100,9 @@ under the License. | W05 | 验证不同类型与 NULL 的 partition/bucket transform | R02、R03、R11 | 功能、正确性、边界 | Iceberg v2 | identity/bucket/truncate/time transform 多列组合，包含 NULL | 数据与 `$partitions` 统计一致；NULL 行可过滤且可继续写入 | | W06 | 验证 required/nullable schema change 与合法写入 | R09、R11 | 异常、正确性 | Iceberg required 列 | 拒绝增加无默认值 required 列和 nullable→required；执行 VALUES/INSERT SELECT 合法写入 | schema change 失败不产生 snapshot；合法写入与 Spark 结果一致 | | W07 | 验证 required 列 NULL 拒绝和 statement 原子性 | R09、R11 | 隔离负向、正确性 | 隔离 Iceberg database | VALUES 写 NULL；多 bucket 源表 INSERT SELECT 混合有效与 NULL 行 | 修复前会错误提交并产生不可读文件；修复后整条语句在 snapshot 发布前拒绝 | -| W08 | 验证 STRING truncate 的 Nullable block 处理 | R03、R05、R12 | 隔离负向、稳定性 | 可重启的隔离 Doris 集群 | nullable STRING INSERT；partition evolution 后 UPDATE 产生 Nullable block | 修复前 BE FATAL；修复后写入成功并保持 NULL 分区语义 | +| W08 | 验证 STRING truncate 的 Nullable block 处理 | R03、R05、R12 | 常规回归、稳定性 | Iceberg v2 | nullable STRING INSERT；partition evolution 后 UPDATE 产生 Nullable block | 写入成功并保持 NULL 分区语义，BE 不发生 FATAL | | W09 | 验证 MERGE 条件动作、多个 NOT MATCHED 与 NULL key 语义 | R02、R03、R05 | 功能、正确性 | Iceberg v2 MOR | identity/bucket 分区间移动、删除、插入、NULL-safe 与普通等值匹配 | 每个源行只选择一个动作，Spark 与 Doris 结果一致 | -| W10 | 验证 MERGE 多源匹配单目标的基数约束 | R13 | 隔离负向、原子性 | Iceberg v2 MOR | 两个源行同时更新一个目标行 | 修复前错误提交重复行；修复后整句拒绝且无新快照和文件 | +| W10 | 验证 MERGE 多源匹配单目标的基数约束 | R13 | 常规回归、原子性 | Iceberg v2 MOR | 两个源行同时更新一个目标行 | 整句拒绝且无新快照和文件 | | W11 | 验证 branch/tag 的写入能力边界 | R04、R14 | 功能、异常、原子性 | 已建立 branch 与 tag | branch INSERT/OVERWRITE；branch 行级 DML 与 tag 写入 | branch 与 main 隔离；不支持操作明确拒绝且引用不变化 | | W12 | 验证多次 Partition Evolution 后覆盖写和历史引用 | R02、R04、R10、R15 | 功能、正确性 | Iceberg v2 | ADD/REPLACE/DROP identity、bucket、truncate、day/hour 后动态覆盖写 | 仅替换当前 spec 命中的分区，tag/branch 和旧 spec 保持可读 | | W13 | 验证 delete files 与覆盖写、演进的交互 | R02、R05、R15 | 正确性、兼容性 | Iceberg v2 MOR | DELETE/UPDATE/MERGE 生成 delete files，再在新旧 spec 上覆盖写 | replacement 行不被旧 delete files 隐藏，历史 tag 不受影响 | @@ -111,10 +111,10 @@ under the License. | W16 | 验证 CTAS、复杂类型、格式和失败清理 | R08、R09、R16 | 功能、异常、兼容性 | 内部多 bucket 源表 | CTAS 到 ORC 分区表；严格转换失败；向 Avro 表写入 | ORC 与 Spark 一致；失败不遗留表或快照；Avro 明确拒绝 | | W17 | 验证 sort order、distribution mode 和多文件 flush | R03、R11、R17 | 正确性、稳定性 | 多 BE Doris | NULL sort key、多列升降序、none/hash/range、低 target file size | 计划包含声明排序，多文件总行数正确，三种分布模式结果一致 | | W18 | 验证并发 MERGE 与 append 的提交不变量 | R11、R13、R17 | 并发、原子性 | 多 BE Doris | readiness barrier 保证两个独立会话同时具备 dispatch 条件；同行更新与互不冲突 append | 同行提交可串行化且基数为一；仅接受 Iceberg validation/commit conflict；非冲突写入无丢失或重复 | -| W19 | 验证 MERGE source projection 进入 truncate transform 的类型安全 | R12、R18 | 隔离负向、稳定性 | 可重启的隔离 Doris 集群 | required STRING truncate 列执行匹配更新与未匹配插入 | 修复前 BE FATAL；修复后 MERGE 成功且物理分区正确 | +| W19 | 验证 MERGE source projection 进入 truncate transform 的类型安全 | R12、R18 | 常规回归、稳定性 | Iceberg v2 MOR | required STRING truncate 列执行匹配更新与未匹配插入 | MERGE 接受 nullable 物理投影中的非 NULL 值，并生成正确物理分区 | ## P0 覆盖检查 -R01-R18 均映射到至少一个 P0 regression。十五个正向 suite 在最后一次成功写入后均由 Spark/Doris 交叉校验同表逻辑结果；transform、rollover、CTAS property 和失败原子性另有物理 metadata 或状态 oracle。稳定性或已确认正确性缺陷使用独立 suite、完整预期输出和显式隔离开关保存复现，避免默认 P0 破坏共享集群或固化错误结果。 +R01-R18 均映射到至少一个 P0 regression。十五个正向 suite 在最后一次成功写入后均由 Spark/Doris 交叉校验同表逻辑结果；transform、rollover、CTAS property 和失败原子性另有物理 metadata 或状态 oracle。W08、W10、W19 已作为常规回归运行；仍可能写出不可读 required-null 文件的 W07 保留隔离开关，避免破坏共享集群。 -本矩阵未覆盖项为 0。COW 行级 DML、branch 行级 DML、tag 写入和 Avro 写入属于当前明确能力边界，均以预期拒绝用例固化错误语义与失败原子性；已确认的产品缺陷均有隔离负向 regression。 +本矩阵未覆盖项为 0。COW 行级 DML、branch 行级 DML、tag 写入和 Avro 写入属于当前明确能力边界，均以预期拒绝用例固化错误语义与失败原子性；仅 required-null 已知缺陷继续使用隔离负向 regression。 diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy index fbb3be22c44834..075bfccad96f8e 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy @@ -16,23 +16,33 @@ // under the License. suite("test_iceberg_write_merge_duplicate_source_negative", - "p0,external,iceberg,external_docker,external_docker_iceberg") { + "p0,external,iceberg,external_docker,external_docker_iceberg,nonConcurrent") { String enabled = context.config.otherConfigs.get("enableIcebergTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { logger.info("disable iceberg test") return } - String knownBugEnabled = context.config.otherConfigs.get("enableIcebergKnownBugTest") - if (knownBugEnabled == null || !knownBugEnabled.equalsIgnoreCase("true")) { - logger.info("skip isolated Iceberg known-bug test") - return - } - String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") String minioPort = context.config.otherConfigs.get("iceberg_minio_port") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String catalogName = "test_iceberg_write_merge_duplicate_source_negative" String dbName = "iceberg_write_merge_duplicate_source_negative_db" + String dockerCommand = context.config.otherConfigs.get("externalDockerCommand") ?: "docker" + String mcImage = "minio/mc:RELEASE.2025-01-17T23-25-50Z" + + def countDataObjects = { String objectPath -> + def stdout = new StringBuilder() + def stderr = new StringBuilder() + def process = new ProcessBuilder("/bin/bash", "-c", + "${dockerCommand} run --rm --entrypoint /bin/sh ${mcImage} -c " + + "'/usr/bin/mc alias set minio http://${externalEnvIp}:${minioPort} " + + "admin password >/dev/null " + + "&& /usr/bin/mc find ${objectPath} --type f | wc -l'").start() + process.consumeProcessOutput(stdout, stderr) + process.waitForOrKill(30000) + assertEquals(0, process.exitValue(), "Failed to list Iceberg data objects: ${stderr}") + return stdout.toString().trim() as long + } sql """drop catalog if exists ${catalogName}""" sql """ @@ -68,35 +78,94 @@ suite("test_iceberg_write_merge_duplicate_source_negative", """ sql """insert into duplicate_source_target values (1, 'A', 'committed')""" + String committedFile = (sql """ + select file_path from duplicate_source_target\$files order by file_path limit 1 + """)[0][0].toString() + int dataDirectoryEnd = committedFile.indexOf('/data/') + '/data'.length() + assertTrue(dataDirectoryEnd >= '/data'.length(), "Unexpected Iceberg data path: ${committedFile}") + String dataObjectPath = committedFile.substring(0, dataDirectoryEnd) + .replaceFirst('^s3a?://warehouse', 'minio/warehouse') + long objectsBefore = countDataObjects(dataObjectPath) + long snapshotsBefore = (sql """select count(*) from duplicate_source_target\$snapshots""")[0][0] as long long filesBefore = (sql """select count(*) from duplicate_source_target\$files""")[0][0] as long // Negative scenario: Iceberg MERGE cardinality permits only one source row - // to update a target row. The entire statement must fail before publishing. + // to update a target row. Tiny blocks and files force a valid row to roll before + // the late duplicate, and the entire statement must still leave no data object behind. + sql """set batch_size = 1""" + // A single pipeline instance preserves the ordered source sequence needed to prove late cleanup. + sql """set parallel_pipeline_task_num = 1""" + sql """set iceberg_write_target_file_size_bytes = 1""" test { sql """ merge into duplicate_source_target t using ( - select 1 as id, 'B' as region, 'first-update' as payload - union all - select 1, 'C', 'second-update' + select id, region, payload + from ( + select 1 as seq, 2 as id, 'B' as region, 'valid-insert' as payload + union all + select 2, 1, 'B', 'first-update' + union all + select 3, 1, 'C', 'second-update' + ) ordered_source + order by seq ) s on t.id = s.id when matched then update set region = s.region, payload = s.payload + when not matched then insert (id, region, payload) + values (s.id, s.region, s.payload) """ - exception "more than one" + exception "multiple source rows matched the same target row" } assertEquals(snapshotsBefore, (sql """select count(*) from duplicate_source_target\$snapshots""")[0][0] as long) assertEquals(filesBefore, (sql """select count(*) from duplicate_source_target\$files""")[0][0] as long) + assertEquals(objectsBefore, countDataObjects(dataObjectPath)) order_qt_duplicate_source_atomic_state """ select id, region, payload from duplicate_source_target order by id """ + + // A sibling delete close can fail only after the data side has closed successfully. The outer + // MERGE still owns and must remove those unpublished data objects. + long siblingFailureObjectsBefore = countDataObjects(dataObjectPath) + try { + GetDebugPoint().enableDebugPointForAllBEs("VIcebergDeleteSink.close.inject_failure") + test { + sql """ + merge into duplicate_source_target t + using ( + select 1 as id, 'B' as region, 'updated' as payload + union all + select 3, 'C', 'inserted' + ) s + on t.id = s.id + when matched then update set + region = s.region, + payload = s.payload + when not matched then insert (id, region, payload) + values (s.id, s.region, s.payload) + """ + exception "injected Iceberg delete close failure" + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("VIcebergDeleteSink.close.inject_failure") + } + assertEquals(snapshotsBefore, + (sql """select count(*) from duplicate_source_target\$snapshots""")[0][0] as long) + assertEquals(filesBefore, + (sql """select count(*) from duplicate_source_target\$files""")[0][0] as long) + assertEquals(siblingFailureObjectsBefore, countDataObjects(dataObjectPath)) + order_qt_sibling_close_atomic_state """ + select id, region, payload + from duplicate_source_target + order by id + """ } diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy index 31bfeb251ab609..8a503d879ace39 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy @@ -18,10 +18,8 @@ suite("test_iceberg_write_merge_truncate_negative", "p0,external,iceberg,external_docker,external_docker_iceberg") { String enabled = context.config.otherConfigs.get("enableIcebergTest") - String crashTestEnabled = context.config.otherConfigs.get("enableIcebergCrashTest") - if (enabled == null || !enabled.equalsIgnoreCase("true") - || crashTestEnabled == null || !crashTestEnabled.equalsIgnoreCase("true")) { - logger.info("disable iceberg crash test") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") return } @@ -67,9 +65,9 @@ suite("test_iceberg_write_merge_truncate_negative", """ sql """insert into merge_truncate_negative values (1, 'alpha', 'before')""" - // WM03-S01: A MERGE source projection is nullable even when every source - // value and the Iceberg target column are NOT NULL. The writer must reject - // an invalid input as a query error and must never terminate a BE. + // WM03-S01: A MERGE source projection is physically nullable even when every source + // value and the Iceberg target column are NOT NULL. The writer must accept those + // non-NULL values without terminating a BE or changing truncate partition routing. sql """ merge into merge_truncate_negative t using ( @@ -92,8 +90,9 @@ suite("test_iceberg_write_merge_truncate_negative", """ // Logical rows cannot prove the MERGE writer used truncate(2) when routing // its updated and inserted records. + // Iceberg names truncate transform fields with the `_trunc` suffix; the width is not part of the name. order_qt_merge_truncate_physical_partitions """ - select distinct hex(struct_element(`partition`, 'partition_value_trunc_2')) + select distinct hex(struct_element(`partition`, 'partition_value_trunc')) from merge_truncate_negative\$partitions order by 1 """ diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy index 25074a0af78d63..a0358aef5b88d9 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy @@ -23,14 +23,6 @@ suite("test_iceberg_write_nullable_truncate_negative", return } - // This opt-in switch isolates a BE-fatal negative scenario from the shared P0 cluster. - // Enable it only in a cluster whose BE processes can be restarted after the suite. - String crashTestEnabled = context.config.otherConfigs.get("enableIcebergCrashTest") - if (crashTestEnabled == null || !crashTestEnabled.equalsIgnoreCase("true")) { - logger.info("skip isolated Iceberg crash regression") - return - } - String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") String minioPort = context.config.otherConfigs.get("iceberg_minio_port") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy index 6d449f4af1e1b4..1fd4e27db7b83e 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy @@ -23,14 +23,6 @@ suite("test_paimon_ctas_atomicity_negative", return } - // CTAS currently creates Paimon metadata before discovering that no Paimon data sink exists. - // Keep this opt-in until the product rejects or rolls back the statement atomically. - String knownBugEnabled = context.config.otherConfigs.get("enablePaimonKnownBugTest") - if (knownBugEnabled == null || !knownBugEnabled.equalsIgnoreCase("true")) { - logger.info("skip isolated Paimon known-bug regression") - return - } - String minioPort = context.config.otherConfigs.get("iceberg_minio_port") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String catalogName = "test_paimon_ctas_atomicity_negative" @@ -67,6 +59,28 @@ suite("test_paimon_ctas_atomicity_negative", exception "PaimonExternalCatalog" } assertEquals(0, (sql """show tables like 'ctas_target'""").size()) + + spark_paimon """ + create table paimon.${dbName}.ctas_target (id int, payload string) + using paimon + """ + // IF NOT EXISTS must remain a no-op even though Paimon does not support the CTAS sink. + sql """ + create table if not exists ctas_target engine=paimon + as select cast(1 as int) as id, cast('candidate' as string) as payload + """ + assertEquals(1, (sql """show tables like 'ctas_target'""").size()) + assertEquals(0, (sql """select * from ctas_target""").size()) + + // An existing non-idempotent target must keep catalog error precedence; no sink can own it. + test { + sql """ + create table ctas_target engine=paimon + as select cast(2 as int) as id, cast('replacement' as string) as payload + """ + exception "already exists" + } + assertEquals(0, (sql """select * from ctas_target""").size()) } finally { spark_paimon """drop table if exists paimon.${dbName}.ctas_target""" sql """drop catalog if exists ${catalogName}""" From 79be3cf57fb3510e728c66b025c9df899b831468 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 29 Jul 2026 21:37:33 +0800 Subject: [PATCH 2/4] [fix](paimon) Correct pruned struct field reads (#66223) - Preserve the full Paimon `RowType` arity when reading projected struct fields. - Add unit coverage for non-leading, non-contiguous, and complete struct projections backed by a binary row. Struct projection entries use original child indexes, but the scanner initialized the nested binary row with the number of projected fields. A projection such as the third child therefore created a one-field row and then accessed index two, producing incorrect reads or an out-of-bounds failure. - `testUnpackNonLeadingStructField` constructs a three-field binary row and projects only original field index 2. The previous implementation fails with `index (2) should < 1`. - `testUnpackNonContiguousStructFields` projects original field indexes 0 and 2. The previous implementation fails with `index (2) should < 2`. - `testUnpackCompleteStruct` protects the existing full-struct behavior. - With this fix restored, all three tests pass with Maven build caching disabled. - Reverted the row-arity fix locally and confirmed that the two pruned-field regression tests fail. - Restored the fix and reran the same test class: 3 tests, 0 failures, 0 errors. - Ran all Paimon scanner test classes with Maven build caching disabled: 15 tests, 0 failures, 0 errors. - Checkstyle passed for the affected reactor modules. (cherry picked from commit 5ede4b69723b331e61989d446ff2ea91c91ee534) --- .../doris/paimon/PaimonColumnValue.java | 7 +- .../doris/paimon/PaimonColumnValueTest.java | 89 +++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index d6db3d4d954f6e..f60bb996703ec8 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -197,11 +197,12 @@ public void unpackMap(List keys, List values) { @Override public void unpackStruct(List structFieldIndex, List values) { - // todo: support pruned struct fields - InternalRow row = record.getRow(idx, structFieldIndex.size()); + RowType rowType = (RowType) dataType; + // Projection entries are original child indexes, so the binary row must keep the full RowType arity. + InternalRow row = record.getRow(idx, rowType.getFieldCount()); for (int i : structFieldIndex) { values.add(new PaimonColumnValue(row, i, dorisType.getChildTypes().get(i), - ((RowType) dataType).getFields().get(i).type(), timeZone)); + rowType.getFields().get(i).type(), timeZone)); } } } diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java new file mode 100644 index 00000000000000..639a2e87f30c18 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ColumnValue; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class PaimonColumnValueTest { + private final RowType paimonStructType = RowType.of( + new DataType[] {new IntType(), new VarCharType(), new BigIntType()}, + new String[] {"a", "b", "c"}); + private final ColumnType dorisStructType = + ColumnType.parseType("s", "struct"); + + @Test + public void testUnpackNonLeadingStructField() { + PaimonColumnValue structValue = createStructValue(); + List values = new ArrayList<>(); + + structValue.unpackStruct(Collections.singletonList(2), values); + + Assert.assertEquals(1, values.size()); + Assert.assertEquals(100L, values.get(0).getLong()); + } + + @Test + public void testUnpackNonContiguousStructFields() { + PaimonColumnValue structValue = createStructValue(); + List values = new ArrayList<>(); + + structValue.unpackStruct(Arrays.asList(0, 2), values); + + Assert.assertEquals(2, values.size()); + Assert.assertEquals(10, values.get(0).getInt()); + Assert.assertEquals(100L, values.get(1).getLong()); + } + + @Test + public void testUnpackCompleteStruct() { + PaimonColumnValue structValue = createStructValue(); + List values = new ArrayList<>(); + + structValue.unpackStruct(Arrays.asList(0, 1, 2), values); + + Assert.assertEquals(3, values.size()); + Assert.assertEquals(10, values.get(0).getInt()); + Assert.assertEquals("x", values.get(1).getString()); + Assert.assertEquals(100L, values.get(2).getLong()); + } + + private PaimonColumnValue createStructValue() { + GenericRow nestedRow = GenericRow.of(10, BinaryString.fromString("x"), 100L); + RowType outerType = RowType.of(new DataType[] {paimonStructType}, new String[] {"s"}); + InternalRow outerRow = new InternalRowSerializer(outerType).toBinaryRow(GenericRow.of(nestedRow)); + return new PaimonColumnValue(outerRow, 0, dorisStructType, paimonStructType, "UTC"); + } +} From f7f771b7e18a87a3e9a3fdc0e554c89974fb1743 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 29 Jul 2026 23:03:30 +0800 Subject: [PATCH 3/4] [branch-4.1](fix) Adapt CTAS test to drop API --- .../nereids/trees/plans/commands/CreateTableCommandTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java index 38324e828aede5..829df5ab0a3898 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommandTest.java @@ -94,7 +94,7 @@ void ctasValidatesSinkBeforePublishingMetadata() throws Exception { Mockito.verify(env, Mockito.never()).dropTable( Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), - Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.anyBoolean(), Mockito.anyBoolean()); } } @@ -127,7 +127,7 @@ void existingCtasTargetPrecedesUnsupportedSinkValidation() throws Exception { Mockito.verify(env, Mockito.never()).dropTable( Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), - Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.anyBoolean(), Mockito.anyBoolean()); } } } From b864c1b06f703c6dd4cf7b86deb980424c1d0cbc Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 30 Jul 2026 15:00:41 +0800 Subject: [PATCH 4/4] [branch-4.1](fix) Backport lakehouse metadata consistency (#66007) Backport the complete 26-commit change set from apache/doris#66007. Source head: 8acb53c794fde3132f399820dacb75f8942bfd5b (cherry picked from commit 235881e18d38adee35aefedd1867289664ef0ed2) --- be/src/core/arena.h | 18 + .../data_type_serde/data_type_array_serde.cpp | 46 +- .../data_type_serde/data_type_map_serde.cpp | 64 +- .../data_type_nullable_serde.cpp | 16 +- .../data_type_struct_serde.cpp | 16 +- be/src/core/data_type_serde/orc_serde_utils.h | 77 +++ be/src/exprs/function/function_struct.cpp | 7 +- be/src/format_v2/table_reader.h | 206 +++++- .../transformer/vorc_transformer_test.cpp | 205 ++++++ be/test/format_v2/table_reader_test.cpp | 592 ++++++++++++++++++ .../java/org/apache/doris/catalog/Type.java | 26 +- .../common/proc/IndexSchemaProcNode.java | 4 +- .../doris/datasource/ExternalTable.java | 10 +- .../doris/datasource/FileQueryScanNode.java | 57 +- .../apache/doris/datasource/FileScanNode.java | 22 +- .../datasource/hive/HMSExternalTable.java | 39 +- .../datasource/hudi/HudiMvccSnapshot.java | 9 +- .../doris/datasource/hudi/HudiUtils.java | 42 +- .../datasource/hudi/source/HudiScanNode.java | 32 +- .../iceberg/IcebergExternalMetaCache.java | 10 +- .../iceberg/IcebergExternalTable.java | 21 +- .../iceberg/IcebergMvccSnapshot.java | 15 + .../iceberg/IcebergSnapshotCacheValue.java | 165 ++++- .../iceberg/IcebergSysExternalTable.java | 19 + .../iceberg/IcebergTransaction.java | 70 ++- .../datasource/iceberg/IcebergUtils.java | 41 +- .../rewrite/RewriteDataFileExecutor.java | 21 +- .../iceberg/rewrite/RewriteGroupTask.java | 23 + .../iceberg/source/IcebergScanNode.java | 147 ++++- .../datasource/mvcc/EmptyMvccSnapshot.java | 4 + .../doris/datasource/mvcc/MvccSnapshot.java | 8 + .../paimon/PaimonExternalTable.java | 73 ++- .../datasource/paimon/PaimonMvccSnapshot.java | 10 + .../paimon/PaimonSnapshotCacheValue.java | 11 + .../paimon/source/PaimonScanNode.java | 28 +- .../paimon/source/PaimonSource.java | 20 +- .../doris/nereids/StatementContext.java | 26 +- .../analyzer/UnboundIcebergTableSink.java | 38 +- .../translator/PhysicalPlanTranslator.java | 15 +- .../nereids/parser/LogicalPlanBuilder.java | 4 + .../nereids/rules/analysis/BindRelation.java | 30 +- .../nereids/rules/analysis/BindSink.java | 51 +- .../rules/expression/check/CheckCast.java | 11 +- .../LogicalFileScanToPhysicalFileScan.java | 3 +- .../LogicalHudiScanToPhysicalHudiScan.java | 3 +- ...DeleteSinkToPhysicalIcebergDeleteSink.java | 1 + ...rgMergeSinkToPhysicalIcebergMergeSink.java | 1 + ...rgTableSinkToPhysicalIcebergTableSink.java | 1 + .../rules/rewrite/PruneFileScanPartition.java | 14 +- .../doris/nereids/trees/expressions/Cast.java | 9 +- .../functions/scalar/CreateNamedStruct.java | 4 +- .../expressions/literal/StructLiteral.java | 14 +- .../plans/commands/IcebergDeleteCommand.java | 14 +- .../plans/commands/IcebergMergeCommand.java | 24 +- .../plans/commands/IcebergUpdateCommand.java | 20 +- .../insert/IcebergDeleteExecutor.java | 7 +- .../insert/IcebergInsertExecutor.java | 7 +- .../commands/insert/IcebergMergeExecutor.java | 6 +- .../insert/InsertIntoTableCommand.java | 3 +- .../plans/commands/insert/InsertUtils.java | 11 + .../trees/plans/logical/LogicalFileScan.java | 116 +++- .../trees/plans/logical/LogicalHudiScan.java | 67 +- .../logical/LogicalIcebergDeleteSink.java | 22 +- .../logical/LogicalIcebergMergeSink.java | 22 +- .../logical/LogicalIcebergTableSink.java | 25 +- .../plans/logical/LogicalSetOperation.java | 6 +- .../plans/physical/PhysicalFileScan.java | 65 +- .../plans/physical/PhysicalHudiScan.java | 34 +- .../physical/PhysicalIcebergDeleteSink.java | 35 +- .../physical/PhysicalIcebergMergeSink.java | 67 +- .../physical/PhysicalIcebergTableSink.java | 47 +- .../PhysicalLazyMaterializeFileScan.java | 2 +- .../doris/nereids/util/TypeCoercionUtils.java | 27 +- .../doris/planner/IcebergDeleteSink.java | 11 +- .../doris/planner/IcebergMergeSink.java | 12 +- .../doris/planner/IcebergTableSink.java | 25 +- .../common/proc/IndexSchemaProcNodeTest.java | 40 ++ .../datasource/FileQueryScanNodeTest.java | 67 ++ .../doris/datasource/hudi/HudiUtilsTest.java | 84 +++ .../iceberg/IcebergDDLAndDMLPlanTest.java | 24 +- .../iceberg/IcebergSysExternalTableTest.java | 48 ++ .../iceberg/IcebergTransactionTest.java | 159 ++++- .../datasource/iceberg/IcebergUtilsTest.java | 107 ++++ .../rewrite/RewriteDataFileExecutorTest.java | 50 ++ .../iceberg/source/IcebergScanNodeTest.java | 317 +++++++++- .../paimon/PaimonExternalTableTest.java | 29 + .../paimon/source/PaimonScanNodeTest.java | 64 +- .../paimon/source/PaimonSourceTest.java | 68 ++ .../doris/external/hms/HmsCatalogTest.java | 15 + .../doris/nereids/StatementContextTest.java | 128 ++++ .../analyzer/UnboundIcebergTableSinkTest.java | 49 ++ .../rules/expression/check/CheckCastTest.java | 10 + .../PhysicalStorageLayerAggregateTest.java | 2 +- .../rewrite/PruneFileScanPartitionTest.java | 48 ++ .../literal/StructLiteralTest.java | 68 ++ .../insert/IcebergDeleteExecutorTest.java | 9 +- .../insert/IcebergMergeExecutorTest.java | 8 +- .../commands/insert/InsertUtilsTest.java | 72 ++- .../plans/logical/LogicalFileScanTest.java | 174 ++++- .../nereids/util/TypeCoercionUtilsTest.java | 75 +++ .../doris/planner/IcebergDeleteSinkTest.java | 8 +- .../doris/planner/IcebergTableSinkTest.java | 120 ++++ .../apache/doris/qe/HmsQueryCacheTest.java | 13 + ...berg_branch_tag_schema_change_extended.out | 1 - .../iceberg/iceberg_branch_tag_operate.out | 1 - .../iceberg/iceberg_query_tag_branch.out | 133 ++-- .../iceberg_schema_change_ddl_with_branch.out | 1 - ...st_iceberg_schema_dual_relation_matrix.out | 35 ++ ...test_iceberg_schema_ref_actions_matrix.out | 43 ++ ...test_iceberg_schema_time_travel_matrix.out | 6 + .../iceberg/test_iceberg_sys_table.out | Bin 41485 -> 41700 bytes ..._paimon_schema_branch_partition_matrix.out | 4 + ...est_paimon_schema_dual_relation_matrix.out | 29 + ...g_branch_tag_schema_change_extended.groovy | 7 +- .../iceberg/iceberg_branch_tag_operate.groovy | 4 +- .../iceberg/iceberg_query_tag_branch.groovy | 39 +- ...eberg_schema_change_ddl_with_branch.groovy | 24 +- ...iceberg_schema_dual_relation_matrix.groovy | 69 +- ...rg_schema_metadata_atomicity_matrix.groovy | 7 +- ...t_iceberg_schema_ref_actions_matrix.groovy | 70 ++- ...t_iceberg_schema_time_travel_matrix.groovy | 33 +- ...imon_schema_branch_partition_matrix.groovy | 15 +- ..._paimon_schema_dual_relation_matrix.groovy | 53 +- .../test_agg_schema_value_drop.groovy | 7 +- .../test_dup_schema_value_drop.groovy | 7 +- 125 files changed, 4871 insertions(+), 647 deletions(-) create mode 100644 be/src/core/data_type_serde/orc_serde_utils.h create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonSourceTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSinkTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartitionTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.out create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.out create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.out create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.out create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.out diff --git a/be/src/core/arena.h b/be/src/core/arena.h index 32859b47b406b0..a9b92aecf9d4b4 100644 --- a/be/src/core/arena.h +++ b/be/src/core/arena.h @@ -24,6 +24,8 @@ #include #include +#include +#include #include #include @@ -333,6 +335,22 @@ class Arena : private boost::noncopyable { return _used_size_no_head + head->used(); } + bool contains(const void* data, size_t size) const { + const auto address = reinterpret_cast(data); + if (data == nullptr || size > std::numeric_limits::max() - address) { + return false; + } + const auto range_end = address + size; + for (const Chunk* chunk = head; chunk != nullptr; chunk = chunk->prev) { + const auto chunk_begin = reinterpret_cast(chunk->begin); + const auto chunk_used_end = reinterpret_cast(chunk->pos); + if (address >= chunk_begin && range_end <= chunk_used_end) { + return true; + } + } + return false; + } + size_t remaining_space_in_current_chunk() const { if (head == nullptr) { return 0; diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index 55c77c56bee6ed..56146b852248a2 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -19,6 +19,8 @@ #include +#include + #include "common/config.h" #include "common/status.h" #include "core/assert_cast.h" @@ -30,6 +32,7 @@ #include "core/data_type/get_least_supertype.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/complex_type_deserialize_util.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/string_ref.h" #include "exprs/function/function_helpers.h" #include "util/jsonb_document.h" @@ -369,15 +372,44 @@ Status DataTypeArraySerDe::write_column_to_orc(const std::string& timezone, cons const auto& array_col = assert_cast(column); const IColumn& nested_column = array_col.get_data(); const auto& offsets = array_col.get_offsets(); + const size_t source_nested_start = start == 0 ? 0 : offsets[start - 1]; + const size_t source_nested_end = end == 0 ? source_nested_start : offsets[end - 1]; + const bool has_masked_row = + null_map != nullptr && std::any_of(null_map->begin() + start, null_map->begin() + end, + [](UInt8 is_null) { return is_null != 0; }); + if (!has_masked_row) { + for (size_t row_id = start; row_id < end; row_id++) { + cur_batch->offsets[row_id - start + 1] = offsets[row_id] - source_nested_start; + } + RETURN_IF_ERROR(nested_serde->write_column_to_orc( + timezone, nested_column, nullptr, cur_batch->elements.get(), source_nested_start, + source_nested_end, arena, options)); + cur_batch->elements->numElements = source_nested_end - source_nested_start; + cur_batch->numElements = end - start; + return Status::OK(); + } + + auto packed_nested_column = nested_column.clone_empty(); + size_t packed_nested_size = 0; for (size_t row_id = start; row_id < end; row_id++) { - size_t offset = offsets[row_id - 1]; + const size_t nested_start = row_id == 0 ? 0 : offsets[row_id - 1]; size_t next_offset = offsets[row_id]; - RETURN_IF_ERROR(nested_serde->write_column_to_orc(timezone, nested_column, nullptr, - cur_batch->elements.get(), offset, - next_offset, arena, options)); - cur_batch->offsets[row_id + 1] = next_offset; - } - cur_batch->elements->numElements = nested_column.size(); + // ORC omits collection payload for absent parent rows, so offsets and child values must + // be compacted together when a nullable ancestor masks a physically populated array. + if (!(*null_map)[row_id]) { + packed_nested_column->insert_range_from(nested_column, nested_start, + next_offset - nested_start); + packed_nested_size += next_offset - nested_start; + } + cur_batch->offsets[row_id - start + 1] = packed_nested_size; + } + RETURN_IF_ERROR(nested_serde->write_column_to_orc(timezone, *packed_nested_column, nullptr, + cur_batch->elements.get(), 0, + packed_nested_size, arena, options)); + // String batches borrow their source bytes, but the packed column is local to this call; + // keep only those borrowed leaves in the write Arena until Writer::add() consumes them. + copy_orc_string_data_to_arena(cur_batch->elements.get(), arena); + cur_batch->elements->numElements = packed_nested_size; cur_batch->numElements = end - start; return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index 1a2e0643d999b6..8d0af094a8c050 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -17,6 +17,8 @@ #include "core/data_type_serde/data_type_map_serde.h" +#include + #include "arrow/array/builder_nested.h" #include "common/config.h" #include "common/exception.h" @@ -26,6 +28,7 @@ #include "core/column/column_map.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/complex_type_deserialize_util.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/string_ref.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" @@ -439,21 +442,56 @@ Status DataTypeMapSerDe::write_column_to_orc(const std::string& timezone, const const ColumnArray::Offsets64& offsets = map_column.get_offsets(); const IColumn& nested_keys_column = map_column.get_keys(); const IColumn& nested_values_column = map_column.get_values(); - for (size_t row_id = start; row_id < end; row_id++) { - size_t offset = offsets[row_id - 1]; - size_t next_offset = offsets[row_id]; - + const size_t source_nested_start = start == 0 ? 0 : offsets[start - 1]; + const size_t source_nested_end = end == 0 ? source_nested_start : offsets[end - 1]; + const bool has_masked_row = + null_map != nullptr && std::any_of(null_map->begin() + start, null_map->begin() + end, + [](UInt8 is_null) { return is_null != 0; }); + if (!has_masked_row) { + for (size_t row_id = start; row_id < end; row_id++) { + cur_batch->offsets[row_id - start + 1] = offsets[row_id] - source_nested_start; + } RETURN_IF_ERROR(key_serde->write_column_to_orc(timezone, nested_keys_column, nullptr, - cur_batch->keys.get(), offset, next_offset, - arena, options)); - RETURN_IF_ERROR(value_serde->write_column_to_orc(timezone, nested_values_column, nullptr, - cur_batch->elements.get(), offset, - next_offset, arena, options)); - - cur_batch->offsets[row_id + 1] = next_offset; + cur_batch->keys.get(), source_nested_start, + source_nested_end, arena, options)); + RETURN_IF_ERROR(value_serde->write_column_to_orc( + timezone, nested_values_column, nullptr, cur_batch->elements.get(), + source_nested_start, source_nested_end, arena, options)); + cur_batch->keys->numElements = source_nested_end - source_nested_start; + cur_batch->elements->numElements = source_nested_end - source_nested_start; + cur_batch->numElements = end - start; + return Status::OK(); } - cur_batch->keys->numElements = nested_keys_column.size(); - cur_batch->elements->numElements = nested_values_column.size(); + + auto packed_keys_column = nested_keys_column.clone_empty(); + auto packed_values_column = nested_values_column.clone_empty(); + size_t packed_nested_size = 0; + for (size_t row_id = start; row_id < end; row_id++) { + const size_t nested_start = row_id == 0 ? 0 : offsets[row_id - 1]; + size_t next_offset = offsets[row_id]; + // ORC omits collection payload for absent parent rows, so keys, values, and offsets must + // share one compacted coordinate space when a nullable ancestor masks a populated map. + if (!(*null_map)[row_id]) { + packed_keys_column->insert_range_from(nested_keys_column, nested_start, + next_offset - nested_start); + packed_values_column->insert_range_from(nested_values_column, nested_start, + next_offset - nested_start); + packed_nested_size += next_offset - nested_start; + } + cur_batch->offsets[row_id - start + 1] = packed_nested_size; + } + RETURN_IF_ERROR(key_serde->write_column_to_orc(timezone, *packed_keys_column, nullptr, + cur_batch->keys.get(), 0, packed_nested_size, + arena, options)); + RETURN_IF_ERROR(value_serde->write_column_to_orc(timezone, *packed_values_column, nullptr, + cur_batch->elements.get(), 0, + packed_nested_size, arena, options)); + // String batches borrow their source bytes, but the packed columns are local to this call; + // keep only those borrowed leaves in the write Arena until Writer::add() consumes them. + copy_orc_string_data_to_arena(cur_batch->keys.get(), arena); + copy_orc_string_data_to_arena(cur_batch->elements.get(), arena); + cur_batch->keys->numElements = packed_nested_size; + cur_batch->elements->numElements = packed_nested_size; cur_batch->numElements = end - start; return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index 72d39645531d2a..a0b6701f0506f6 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -493,7 +493,17 @@ Status DataTypeNullableSerDe::write_column_to_orc(const std::string& timezone, const auto& column_nullable = assert_cast(column); orc_col_batch->hasNulls = true; const auto& null_map_tmp = column_nullable.get_null_map_data(); - auto orc_null_map = revert_null_map(&null_map_tmp, start, end); + const NullMap* effective_null_map = &null_map_tmp; + NullMap combined_null_map; + if (null_map != nullptr && null_map != &null_map_tmp) { + DORIS_CHECK(null_map->size() == null_map_tmp.size()); + combined_null_map.assign(null_map_tmp.begin(), null_map_tmp.end()); + for (size_t row = start; row < static_cast(end); ++row) { + combined_null_map[row] |= (*null_map)[row]; + } + effective_null_map = &combined_null_map; + } + auto orc_null_map = revert_null_map(effective_null_map, start, end); // orc_col_batch->notNull.data() must add 'start' (+ start), // because orc_col_batch->notNull.data() begins at 0 // orc_null_map.data() do not need add 'start' (+ start), @@ -501,8 +511,8 @@ Status DataTypeNullableSerDe::write_column_to_orc(const std::string& timezone, memcpy(orc_col_batch->notNull.data() + start, orc_null_map.data(), end - start); RETURN_IF_ERROR(nested_serde->write_column_to_orc(timezone, column_nullable.get_nested_column(), - &column_nullable.get_null_map_data(), - orc_col_batch, start, end, arena, options)); + effective_null_map, orc_col_batch, start, end, + arena, options)); return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index e03aa9cb963f1f..c1baa50fceb609 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -447,12 +447,18 @@ Status DataTypeStructSerDe::write_column_to_orc(const std::string& timezone, con const FormatOptions& options) const { auto* cur_batch = dynamic_cast(orc_col_batch); const auto& struct_col = assert_cast(column); - for (auto row_id = start; row_id < end; row_id++) { - for (int i = 0; i < struct_col.tuple_size(); ++i) { - RETURN_IF_ERROR(elem_serdes_ptrs[i]->write_column_to_orc( - timezone, struct_col.get_column(i), nullptr, cur_batch->fields[i], row_id, - row_id + 1, arena, options)); + for (int i = 0; i < struct_col.tuple_size(); ++i) { + if (null_map != nullptr) { + // ORC child writers filter values with the child's presence stream, not only the + // incoming parent mask, so propagate parent nulls into every child batch. + cur_batch->fields[i]->hasNulls = true; + for (auto row_id = start; row_id < end; ++row_id) { + cur_batch->fields[i]->notNull[row_id] = !(*null_map)[row_id]; + } } + RETURN_IF_ERROR(elem_serdes_ptrs[i]->write_column_to_orc(timezone, struct_col.get_column(i), + null_map, cur_batch->fields[i], + start, end, arena, options)); } cur_batch->numElements = end - start; diff --git a/be/src/core/data_type_serde/orc_serde_utils.h b/be/src/core/data_type_serde/orc_serde_utils.h new file mode 100644 index 00000000000000..57df53f4feb0a1 --- /dev/null +++ b/be/src/core/data_type_serde/orc_serde_utils.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "core/arena.h" + +namespace doris { + +inline void copy_orc_string_data_to_arena(orc::ColumnVectorBatch* batch, Arena& arena) { + if (auto* strings = dynamic_cast(batch)) { + size_t total_size = 0; + for (size_t i = 0; i < strings->numElements; ++i) { + const size_t length = static_cast(strings->length[i]); + // Some serdes already allocate their payload in this Arena; copying it again would + // double the serialized string memory without extending its lifetime. + if (length > 0 && !arena.contains(strings->data[i], length)) { + total_size += length; + } + } + char* cursor = total_size == 0 ? nullptr : arena.alloc(total_size); + for (size_t i = 0; i < strings->numElements; ++i) { + const size_t length = static_cast(strings->length[i]); + const char* source = strings->data[i]; + if (length > 0 && !arena.contains(source, length)) { + std::memcpy(cursor, source, length); + strings->data[i] = cursor; + cursor += length; + } else if (length == 0) { + static char empty_string_sentinel = '\0'; + // ORC treats a null data pointer as absent even when length is zero, so retain a + // stable non-null address to keep empty strings in min/max statistics. + strings->data[i] = &empty_string_sentinel; + } + } + return; + } + if (auto* structure = dynamic_cast(batch)) { + for (orc::ColumnVectorBatch* field : structure->fields) { + copy_orc_string_data_to_arena(field, arena); + } + return; + } + if (auto* list = dynamic_cast(batch)) { + copy_orc_string_data_to_arena(list->elements.get(), arena); + return; + } + if (auto* map = dynamic_cast(batch)) { + copy_orc_string_data_to_arena(map->keys.get(), arena); + copy_orc_string_data_to_arena(map->elements.get(), arena); + return; + } + if (auto* union_batch = dynamic_cast(batch)) { + for (orc::ColumnVectorBatch* child : union_batch->children) { + copy_orc_string_data_to_arena(child, arena); + } + } +} + +} // namespace doris diff --git a/be/src/exprs/function/function_struct.cpp b/be/src/exprs/function/function_struct.cpp index 5d490c58fc488c..3a90bed99ce5e6 100644 --- a/be/src/exprs/function/function_struct.cpp +++ b/be/src/exprs/function/function_struct.cpp @@ -118,7 +118,9 @@ struct StructImpl { static void check_number_of_arguments(size_t number_of_arguments) {} static DataTypePtr get_return_type_impl(const DataTypes& arguments) { - return std::make_shared(make_nullable(arguments)); + // FE plans the same child-nullability contract, so widening fields here makes the + // serialized plan disagree with BE's inferred result type. + return std::make_shared(arguments); } }; @@ -139,7 +141,8 @@ struct NamedStructImpl { data_types[i] = arguments[even_idx]; even_idx += 2; } - return std::make_shared(make_nullable(data_types)); + // Preserve value nullability just like struct(); field-name arguments do not affect it. + return std::make_shared(data_types); } }; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index baf2feb3c4f454..6d679ec2d797cf 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1152,7 +1152,8 @@ class TableReader { return _detach_column(std::move(column)); } - static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type) { + static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type, + const NullMap* nullable_parent_null_map = nullptr) { DORIS_CHECK(column != nullptr); DORIS_CHECK(column->get() != nullptr); DORIS_CHECK(table_type != nullptr); @@ -1162,13 +1163,28 @@ class TableReader { const auto& nested_type = assert_cast(*table_type).get_nested_type(); if (!(*column)->is_nullable()) { - RETURN_IF_ERROR(_align_column_nullability(column, nested_type)); + RETURN_IF_ERROR( + _align_column_nullability(column, nested_type, nullable_parent_null_map)); *column = make_nullable(*column); return Status::OK(); } const auto& nullable_column = assert_cast(**column); ColumnPtr nested_column = nullable_column.get_nested_column_ptr(); - RETURN_IF_ERROR(_align_column_nullability(&nested_column, nested_type)); + NullMap combined_null_map; + const NullMap* nested_parent_null_map = &nullable_column.get_null_map_data(); + if (nullable_parent_null_map != nullptr) { + const auto& own_null_map = nullable_column.get_null_map_data(); + DORIS_CHECK(nullable_parent_null_map->size() == own_null_map.size()); + // Required descendants are hidden when either this nullable container or any + // inherited nullable ancestor masks the row, so preserve the union recursively. + combined_null_map.resize(own_null_map.size()); + for (size_t i = 0; i < own_null_map.size(); ++i) { + combined_null_map[i] = own_null_map[i] || (*nullable_parent_null_map)[i]; + } + nested_parent_null_map = &combined_null_map; + } + RETURN_IF_ERROR( + _align_column_nullability(&nested_column, nested_type, nested_parent_null_map)); *column = ColumnNullable::create(nested_column, nullable_column.get_null_map_column_ptr()); return Status::OK(); @@ -1176,11 +1192,24 @@ class TableReader { if ((*column)->is_nullable()) { const auto& nullable_column = assert_cast(**column); if (nullable_column.has_null()) { - return Status::InternalError( - "Default expression produced NULL for non-nullable table column"); + const auto& null_map = nullable_column.get_null_map_data(); + if (nullable_parent_null_map == nullptr || + nullable_parent_null_map->size() != null_map.size()) { + return Status::InternalError( + "Default expression produced NULL for non-nullable table column"); + } + for (size_t i = 0; i < null_map.size(); ++i) { + // A required child may contain a physical NULL placeholder only when its + // nullable parent masks that row from the logical value. + if (null_map[i] && !(*nullable_parent_null_map)[i]) { + return Status::InternalError( + "Default expression produced NULL for non-nullable table column"); + } + } } ColumnPtr nested_column = nullable_column.get_nested_column_ptr(); - RETURN_IF_ERROR(_align_column_nullability(&nested_column, table_type)); + RETURN_IF_ERROR(_align_column_nullability(&nested_column, table_type, + nullable_parent_null_map)); *column = nested_column; return Status::OK(); } @@ -1206,8 +1235,8 @@ class TableReader { Columns columns = struct_column.get_columns_copy(); DORIS_CHECK(columns.size() == struct_type->get_elements().size()); for (size_t i = 0; i < columns.size(); ++i) { - RETURN_IF_ERROR( - _align_column_nullability(&columns[i], struct_type->get_element(i))); + RETURN_IF_ERROR(_align_column_nullability(&columns[i], struct_type->get_element(i), + nullable_parent_null_map)); } *column = ColumnStruct::create(columns); return Status::OK(); @@ -1262,7 +1291,12 @@ class TableReader { Block cast_block; cast_block.insert({*column, input_type, column_name}); auto slot_ref = VSlotRef::create_shared(0, 0, -1, input_type, column_name); - auto cast_expr = Cast::create_shared(table_type); + // Preserve the source null map through conversion; the caller validates and unwraps it + // against a required table field after the value conversion finishes. + const auto cast_target_type = input_type->is_nullable() && !table_type->is_nullable() + ? make_nullable(table_type) + : table_type; + auto cast_expr = Cast::create_shared(cast_target_type); cast_expr->add_child(std::move(slot_ref)); auto cast_ctx = VExprContext::create_shared(std::move(cast_expr)); RowDescriptor row_desc; @@ -1274,23 +1308,24 @@ class TableReader { return Status::OK(); } - Status _materialize_present_child_mapping_column(const ColumnMapping& mapping, - const ColumnPtr& file_column, - const size_t rows, ColumnPtr* column) { + Status _materialize_present_child_mapping_column( + const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, + ColumnPtr* column, const NullMap* nullable_parent_null_map = nullptr) { DORIS_CHECK(column != nullptr); DORIS_CHECK(mapping.file_type != nullptr); DORIS_CHECK(mapping.table_type != nullptr); *column = file_column; if (!mapping.is_trivial) { if (!mapping.child_mappings.empty()) { - RETURN_IF_ERROR( - _materialize_complex_mapping_column(mapping, *column, rows, column)); + RETURN_IF_ERROR(_materialize_complex_mapping_column(mapping, *column, rows, column, + nullable_parent_null_map)); } else { RETURN_IF_ERROR(_cast_column_to_type(column, mapping.file_type, mapping.table_type, mapping.file_column_name)); } } - RETURN_IF_ERROR(_align_column_nullability(column, mapping.table_type)); + RETURN_IF_ERROR( + _align_column_nullability(column, mapping.table_type, nullable_parent_null_map)); return Status::OK(); } @@ -1367,19 +1402,23 @@ class TableReader { Status _materialize_complex_mapping_column(const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, - ColumnPtr* column) { + ColumnPtr* column, + const NullMap* nullable_parent_null_map = nullptr) { DORIS_CHECK(mapping.table_type != nullptr); DORIS_CHECK(file_column.get() != nullptr); const auto table_type = remove_nullable(mapping.table_type); switch (table_type->get_primitive_type()) { case TYPE_STRUCT: - RETURN_IF_ERROR(_materialize_struct_mapping_column(mapping, file_column, rows, column)); + RETURN_IF_ERROR(_materialize_struct_mapping_column(mapping, file_column, rows, column, + nullable_parent_null_map)); break; case TYPE_ARRAY: - RETURN_IF_ERROR(_materialize_array_mapping_column(mapping, file_column, rows, column)); + RETURN_IF_ERROR(_materialize_array_mapping_column(mapping, file_column, rows, column, + nullable_parent_null_map)); break; case TYPE_MAP: - RETURN_IF_ERROR(_materialize_map_mapping_column(mapping, file_column, rows, column)); + RETURN_IF_ERROR(_materialize_map_mapping_column(mapping, file_column, rows, column, + nullable_parent_null_map)); break; default: *column = _detach_column(file_column); @@ -1452,9 +1491,39 @@ class TableReader { return column.get(); } + template + static const NullMap* _project_collection_parent_null_map( + const NullMap* container_null_map, const NullMap* ancestor_null_map, const size_t rows, + const Offsets& offsets, const size_t child_rows, NullMap* const projected_null_map) { + if (container_null_map == nullptr && ancestor_null_map == nullptr) { + return nullptr; + } + DORIS_CHECK(container_null_map == nullptr || container_null_map->size() == rows); + DORIS_CHECK(ancestor_null_map == nullptr || ancestor_null_map->size() == rows); + DORIS_CHECK(offsets.size() == rows); + projected_null_map->resize(child_rows); + std::fill(projected_null_map->begin(), projected_null_map->end(), 0); + size_t begin = 0; + for (size_t row = 0; row < rows; ++row) { + const size_t end = offsets[row]; + const bool hidden = (container_null_map != nullptr && (*container_null_map)[row]) || + (ancestor_null_map != nullptr && (*ancestor_null_map)[row]); + if (hidden) { + // Collection masks use row coordinates; descendants need the same invariant + // projected through offsets so hidden physical payload cannot fail validation. + std::fill(projected_null_map->begin() + begin, projected_null_map->begin() + end, + 1); + } + begin = end; + } + DORIS_CHECK(begin == child_rows); + return projected_null_map; + } + Status _materialize_struct_mapping_column(const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, - ColumnPtr* column) { + ColumnPtr* column, + const NullMap* nullable_parent_null_map = nullptr) { DORIS_CHECK(mapping.table_type != nullptr); const auto* table_type = assert_cast(remove_nullable(mapping.table_type).get()); @@ -1465,6 +1534,33 @@ class TableReader { const auto* file_struct = assert_cast(nested_file_column); DORIS_CHECK(table_type->get_elements().size() == mapping.child_mappings.size()); + NullMap combined_parent_null_map; + const NullMap* descendant_parent_null_map = nullable_parent_null_map; + if (parent_null_map != nullptr) { + DORIS_CHECK(parent_null_map->size() == rows); + if (nullable_parent_null_map != nullptr) { + DORIS_CHECK(nullable_parent_null_map->size() == rows); + } + if (!mapping.table_type->is_nullable()) { + for (size_t i = 0; i < rows; ++i) { + // A required nested container may drop its own NULL only when an ancestor + // already hides that row; otherwise physical defaults become visible values. + if ((*parent_null_map)[i] && + (nullable_parent_null_map == nullptr || !(*nullable_parent_null_map)[i])) { + return Status::InternalError( + "Source struct contains NULL for non-nullable table column"); + } + } + } + combined_parent_null_map.resize(rows); + for (size_t i = 0; i < rows; ++i) { + combined_parent_null_map[i] = + (*parent_null_map)[i] || + (nullable_parent_null_map != nullptr && (*nullable_parent_null_map)[i]); + } + descendant_parent_null_map = &combined_parent_null_map; + } + Columns child_columns; child_columns.reserve(mapping.child_mappings.size()); const auto file_ordered_children = @@ -1474,20 +1570,23 @@ class TableReader { for (const auto* child_mapping : table_ordered_children) { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { - child_columns.push_back( + ColumnPtr child_column = (child_mapping->initial_default_column ? child_mapping->initial_default_column->clone_resized(rows) : child_mapping->table_type ->create_column_const_with_default_value(rows)) - ->convert_to_full_column_if_const()); + ->convert_to_full_column_if_const(); + RETURN_IF_ERROR(_align_column_nullability(&child_column, child_mapping->table_type, + descendant_parent_null_map)); + child_columns.push_back(std::move(child_column)); continue; } const auto file_child_idx = _file_child_ordinal_for_mapping(mapping, *child_mapping, file_ordered_children); DORIS_CHECK(file_child_idx < file_struct->get_columns().size()); ColumnPtr child_column = file_struct->get_column_ptr(file_child_idx); - RETURN_IF_ERROR(_materialize_present_child_mapping_column(*child_mapping, child_column, - rows, &child_column)); + RETURN_IF_ERROR(_materialize_present_child_mapping_column( + *child_mapping, child_column, rows, &child_column, descendant_parent_null_map)); child_columns.push_back(std::move(child_column)); } MutableColumns mutable_child_columns; @@ -1515,17 +1614,41 @@ class TableReader { Status _materialize_array_mapping_column(const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, - ColumnPtr* column) { + ColumnPtr* column, + const NullMap* nullable_parent_null_map = nullptr) { DORIS_CHECK(mapping.child_mappings.size() == 1); const auto full_file_column = file_column->convert_to_full_column_if_const(); const NullMap* parent_null_map = nullptr; const auto* nested_file_column = _nested_column_if_nullable(full_file_column, &parent_null_map); + if (parent_null_map != nullptr && !mapping.table_type->is_nullable()) { + DORIS_CHECK(parent_null_map->size() == rows); + if (nullable_parent_null_map != nullptr) { + DORIS_CHECK(nullable_parent_null_map->size() == rows); + } + for (size_t i = 0; i < rows; ++i) { + // ARRAY row masks cannot be forwarded to elements because they use different + // coordinates, so validate the container before dropping its nullable wrapper. + if ((*parent_null_map)[i] && + (nullable_parent_null_map == nullptr || !(*nullable_parent_null_map)[i])) { + return Status::InternalError( + "Source array contains NULL for non-nullable table column"); + } + } + } const auto* file_array = assert_cast(nested_file_column); ColumnPtr nested_column = file_array->get_data_ptr(); - const auto& element_mapping = mapping.child_mappings[0]; + auto element_mapping = mapping.child_mappings[0]; + // Keep the descriptor type for schema matching. ARRAY's nullable element wrapper is a + // storage invariant, so add it only at the materialization boundary. + element_mapping.table_type = make_nullable(element_mapping.table_type); + NullMap descendant_parent_null_map; + const NullMap* descendant_parent_null_map_ptr = _project_collection_parent_null_map( + parent_null_map, nullable_parent_null_map, rows, file_array->get_offsets(), + nested_column->size(), &descendant_parent_null_map); RETURN_IF_ERROR(_materialize_present_child_mapping_column( - element_mapping, nested_column, nested_column->size(), &nested_column)); + element_mapping, nested_column, nested_column->size(), &nested_column, + descendant_parent_null_map_ptr)); auto offsets_column = file_array->get_offsets_ptr()->convert_to_full_column_if_const(); auto result = ColumnArray::create(IColumn::mutate(std::move(nested_column)), IColumn::mutate(std::move(offsets_column))); @@ -1548,14 +1671,35 @@ class TableReader { Status _materialize_map_mapping_column(const ColumnMapping& mapping, const ColumnPtr& file_column, const size_t rows, - ColumnPtr* column) { + ColumnPtr* column, + const NullMap* nullable_parent_null_map = nullptr) { const auto full_file_column = file_column->convert_to_full_column_if_const(); const NullMap* parent_null_map = nullptr; const auto* nested_file_column = _nested_column_if_nullable(full_file_column, &parent_null_map); + if (parent_null_map != nullptr && !mapping.table_type->is_nullable()) { + DORIS_CHECK(parent_null_map->size() == rows); + if (nullable_parent_null_map != nullptr) { + DORIS_CHECK(nullable_parent_null_map->size() == rows); + } + for (size_t i = 0; i < rows; ++i) { + // MAP row masks cannot be forwarded to entries because they use different + // coordinates, so validate the container before dropping its nullable wrapper. + if ((*parent_null_map)[i] && + (nullable_parent_null_map == nullptr || !(*nullable_parent_null_map)[i])) { + return Status::InternalError( + "Source map contains NULL for non-nullable table column"); + } + } + } const auto* file_map = assert_cast(nested_file_column); ColumnPtr key_column = file_map->get_keys_ptr(); ColumnPtr value_column = file_map->get_values_ptr(); + DORIS_CHECK(key_column->size() == value_column->size()); + NullMap descendant_parent_null_map; + const NullMap* descendant_parent_null_map_ptr = _project_collection_parent_null_map( + parent_null_map, nullable_parent_null_map, rows, file_map->get_offsets(), + key_column->size(), &descendant_parent_null_map); const ColumnMapping* key_mapping = nullptr; const ColumnMapping* value_mapping = nullptr; @@ -1572,11 +1716,13 @@ class TableReader { if (key_mapping != nullptr) { RETURN_IF_ERROR(_materialize_present_child_mapping_column( - *key_mapping, key_column, key_column->size(), &key_column)); + *key_mapping, key_column, key_column->size(), &key_column, + descendant_parent_null_map_ptr)); } if (value_mapping != nullptr) { RETURN_IF_ERROR(_materialize_present_child_mapping_column( - *value_mapping, value_column, value_column->size(), &value_column)); + *value_mapping, value_column, value_column->size(), &value_column, + descendant_parent_null_map_ptr)); } auto offsets_column = file_map->get_offsets_ptr()->convert_to_full_column_if_const(); auto result = ColumnMap::create(IColumn::mutate(std::move(key_column)), diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp index 4ea14766356d15..124977d47838cb 100644 --- a/be/test/format/transformer/vorc_transformer_test.cpp +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -19,13 +19,22 @@ #include +#include + #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "format/table/iceberg/schema_parser.h" #include "io/fs/local_file_system.h" #include "runtime/runtime_state.h" @@ -47,6 +56,41 @@ class VOrcTransformerTest : public testing::Test { std::shared_ptr _fs; }; +TEST(OrcSerdeUtilsTest, CopiesOnlyBorrowedStringData) { + Arena arena; + char* arena_owned = arena.alloc(5); + std::memcpy(arena_owned, "owned", 5); + std::string borrowed = "borrowed"; + + orc::StringVectorBatch batch(2, *orc::getDefaultPool()); + batch.numElements = 2; + batch.data[0] = arena_owned; + batch.length[0] = 5; + batch.data[1] = borrowed.data(); + batch.length[1] = borrowed.size(); + const size_t used_before_copy = arena.used_size(); + + copy_orc_string_data_to_arena(&batch, arena); + + EXPECT_EQ(arena_owned, batch.data[0]); + EXPECT_NE(borrowed.data(), batch.data[1]); + EXPECT_EQ("borrowed", std::string(batch.data[1], batch.length[1])); + EXPECT_EQ(used_before_copy + borrowed.size(), arena.used_size()); +} + +TEST(OrcSerdeUtilsTest, PreservesEmptyStringAsPresentValue) { + Arena arena; + orc::StringVectorBatch batch(1, *orc::getDefaultPool()); + batch.numElements = 1; + batch.data[0] = const_cast(""); + batch.length[0] = 0; + + copy_orc_string_data_to_arena(&batch, arena); + + EXPECT_NE(batch.data[0], nullptr); + EXPECT_EQ(batch.length[0], 0); +} + TEST_F(VOrcTransformerTest, CollectsBoundsForTopLevelFieldAfterStruct) { auto int_type = std::make_shared(); auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"a"}); @@ -104,4 +148,165 @@ TEST_F(VOrcTransformerTest, CollectsBoundsForTopLevelFieldAfterStruct) { EXPECT_EQ("hello", stats.upper_bounds.at(3)); } +TEST_F(VOrcTransformerTest, PreservesNullableArrayStructChildPositions) { + const auto int_type = make_nullable(std::make_shared()); + const auto string_type = make_nullable(std::make_shared()); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {int_type, string_type}, Strings {"i_info", "s_info"})); + const auto array_type = make_nullable(std::make_shared(struct_type)); + VExprContextSPtrs output_exprs = MockSlotRef::create_mock_contexts(DataTypes {array_type}); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("UTC"); + VOrcTransformer transformer(&state, file_writer.get(), output_exprs, "", {"ss_info"}, false, + TFileCompressType::PLAIN, nullptr, _fs); + ASSERT_TRUE(transformer.open().ok()); + + auto ints = ColumnInt32::create(); + ints->get_data().assign({1, 2, 3, 4, 0, 0, 5, 6, 0, 0, 0, 0, 0, 8, 9, 10}); + auto strings = ColumnString::create(); + for (const auto& value : std::vector { + "doris1", "nereids1", "doris-nereids-1", "doris-nereids-4", "", "", + "doris-nereids-5", "doris7", "", "", "", "", "", "doris8", "doris9", "doris10"}) { + strings->insert_data(value.data(), value.size()); + } + MutableColumns struct_children; + struct_children.push_back(ColumnNullable::create(std::move(ints), ColumnUInt8::create(16, 0))); + struct_children.push_back( + ColumnNullable::create(std::move(strings), ColumnUInt8::create(16, 0))); + auto element_null_map = ColumnUInt8::create(); + element_null_map->get_data().assign({0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0}); + auto elements = ColumnNullable::create(ColumnStruct::create(std::move(struct_children)), + std::move(element_null_map)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().assign({3, 4, 4, 7, 10, 13, 13, 14, 16}); + auto array_null_map = ColumnUInt8::create(); + array_null_map->get_data().assign({0, 0, 0, 0, 0, 0, 1, 0, 0}); + auto array = + ColumnNullable::create(ColumnArray::create(std::move(elements), std::move(offsets)), + std::move(array_null_map)); + + Block block; + block.insert(ColumnWithTypeAndName(std::move(array), array_type, "ss_info")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + ::orc::ReaderOptions reader_options; + auto reader = ::orc::createReader( + ::orc::readLocalFile(_file_path, reader_options.getReaderMetrics()), reader_options); + auto row_reader = reader->createRowReader(); + auto batch = row_reader->createRowBatch(9); + ASSERT_TRUE(row_reader->next(*batch)); + const auto& root = dynamic_cast(*batch); + const auto& list = dynamic_cast(*root.fields[0]); + const auto& element_struct = dynamic_cast(*list.elements); + const auto& element_ints = + dynamic_cast(*element_struct.fields[0]); + const auto& element_strings = + dynamic_cast(*element_struct.fields[1]); + ASSERT_EQ(element_struct.numElements, 16); + EXPECT_FALSE(element_struct.notNull[4]); + EXPECT_FALSE(element_struct.notNull[5]); + EXPECT_TRUE(element_struct.notNull[6]); + EXPECT_EQ(element_ints.data[6], 5); + EXPECT_EQ(std::string(element_strings.data[6], element_strings.length[6]), "doris-nereids-5"); + EXPECT_EQ(element_ints.data[7], 6); + EXPECT_EQ(element_ints.data[13], 8); + EXPECT_EQ(element_ints.data[14], 9); + EXPECT_EQ(element_ints.data[15], 10); +} + +TEST_F(VOrcTransformerTest, CompactsCollectionsMaskedByNullableStruct) { + const auto int_type = make_nullable(std::make_shared()); + const auto string_type = make_nullable(std::make_shared()); + const auto array_type = std::make_shared(int_type); + const auto map_type = std::make_shared(string_type, string_type); + const auto struct_type = make_nullable(std::make_shared( + DataTypes {array_type, map_type}, Strings {"items", "properties"})); + VExprContextSPtrs output_exprs = MockSlotRef::create_mock_contexts(DataTypes {struct_type}); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + state.set_timezone("UTC"); + VOrcTransformer transformer(&state, file_writer.get(), output_exprs, "", {"payload"}, false, + TFileCompressType::PLAIN, nullptr, _fs); + ASSERT_TRUE(transformer.open().ok()); + + auto array_values = ColumnInt32::create(); + array_values->get_data().assign({10, 11, 20}); + auto array_elements = + ColumnNullable::create(std::move(array_values), ColumnUInt8::create(3, 0)); + auto array_offsets = ColumnArray::ColumnOffsets::create(); + array_offsets->get_data().assign({2, 3}); + auto array_column = ColumnArray::create(std::move(array_elements), std::move(array_offsets)); + + auto map_keys = ColumnString::create(); + map_keys->insert_data("hidden-key-1", 12); + map_keys->insert_data("hidden-key-2", 12); + map_keys->insert_default(); + map_keys->insert_data("z", 1); + auto map_values = ColumnString::create(); + map_values->insert_data("hidden-value-1", 14); + map_values->insert_data("hidden-value-2", 14); + map_values->insert_default(); + map_values->insert_data("z", 1); + auto map_offsets = ColumnArray::ColumnOffsets::create(); + map_offsets->get_data().assign({2, 4}); + auto map_column = ColumnMap::create( + ColumnNullable::create(std::move(map_keys), ColumnUInt8::create(4, 0)), + ColumnNullable::create(std::move(map_values), ColumnUInt8::create(4, 0)), + std::move(map_offsets)); + + MutableColumns children; + children.push_back(std::move(array_column)); + children.push_back(std::move(map_column)); + auto parent_null_map = ColumnUInt8::create(); + parent_null_map->get_data().assign({1, 0}); + auto payload = ColumnNullable::create(ColumnStruct::create(std::move(children)), + std::move(parent_null_map)); + Block block; + block.insert(ColumnWithTypeAndName(std::move(payload), struct_type, "payload")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + ::orc::ReaderOptions options; + auto reader = ::orc::createReader(::orc::readLocalFile(_file_path, options.getReaderMetrics()), + options); + auto row_reader = reader->createRowReader(); + auto batch = row_reader->createRowBatch(2); + ASSERT_TRUE(row_reader->next(*batch)); + const auto& root = dynamic_cast(*batch); + const auto& payload_batch = dynamic_cast(*root.fields[0]); + const auto& list = dynamic_cast(*payload_batch.fields[0]); + const auto& list_values = dynamic_cast(*list.elements); + const auto& map = dynamic_cast(*payload_batch.fields[1]); + const auto& map_keys_batch = dynamic_cast(*map.keys); + const auto& map_values_batch = dynamic_cast(*map.elements); + + EXPECT_EQ(list.offsets[0], 0); + EXPECT_EQ(list.offsets[1], 0); + EXPECT_EQ(list.offsets[2], 1); + EXPECT_EQ(list_values.data[0], 20); + EXPECT_EQ(map.offsets[0], 0); + EXPECT_EQ(map.offsets[1], 0); + EXPECT_EQ(map.offsets[2], 2); + EXPECT_EQ(std::string(map_keys_batch.data[0], map_keys_batch.length[0]), ""); + EXPECT_EQ(std::string(map_keys_batch.data[1], map_keys_batch.length[1]), "z"); + EXPECT_EQ(std::string(map_values_batch.data[0], map_values_batch.length[0]), ""); + EXPECT_EQ(std::string(map_values_batch.data[1], map_values_batch.length[1]), "z"); + + auto statistics = reader->getStatistics(); + const auto* payload_type = reader->getType().getSubtype(0); + const auto* map_type_node = payload_type->getSubtype(1); + const auto* key_statistics = dynamic_cast( + statistics->getColumnStatistics(map_type_node->getSubtype(0)->getColumnId())); + ASSERT_NE(key_statistics, nullptr); + ASSERT_TRUE(key_statistics->hasMinimum()); + EXPECT_EQ(key_statistics->getMinimum(), ""); + EXPECT_EQ(key_statistics->getMaximum(), "z"); +} + } // namespace doris diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index bb35fd3926afb0..bb0db9329a2a1c 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -972,6 +972,10 @@ class TableReaderCharVarcharTestHelper final : public TableReader { class TableReaderCastTestHelper final : public TableReader { public: using TableReader::_cast_column_to_type; + using TableReader::_materialize_array_mapping_column; + using TableReader::_materialize_map_mapping_column; + using TableReader::_materialize_present_child_mapping_column; + using TableReader::_materialize_struct_mapping_column; }; TEST(TableReaderTest, TruncateCharOrVarcharPredicateOnlyAppliesToParquetStringWidthMismatch) { @@ -2664,6 +2668,526 @@ TEST(TableReaderTest, ComplexRematerializeCastsNonNullableScalarChildWithNullabl EXPECT_EQ(child_values.get_element(1), 20); } +TEST(TableReaderTest, ComplexRematerializeCastsNullableScalarChildToRequiredTableType) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + auto values = ColumnInt32::create(); + values->insert_value(10); + values->insert_value(20); + ColumnPtr file_column = ColumnNullable::create(std::move(values), ColumnUInt8::create(2, 0)); + ColumnMapping mapping; + mapping.file_column_name = "struct_column.a"; + mapping.file_type = nullable_int_type; + mapping.table_type = bigint_type; + mapping.is_trivial = false; + + ColumnPtr result_column; + const auto status = reader._materialize_present_child_mapping_column(mapping, file_column, 2, + &result_column); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(result_column->is_nullable()); + const auto& result_values = assert_cast(*result_column); + EXPECT_EQ(result_values.get_element(0), 10); + EXPECT_EQ(result_values.get_element(1), 20); +} + +TEST(TableReaderTest, ComplexRematerializeAllowsRequiredChildNullMaskedByParent) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto file_struct_type = make_nullable( + std::make_shared(DataTypes {nullable_int_type}, Strings {"a"})); + const auto table_struct_type = + make_nullable(std::make_shared(DataTypes {bigint_type}, Strings {"a"})); + + ColumnMapping child_mapping; + child_mapping.file_local_id = 0; + child_mapping.file_column_name = "struct_column.a"; + child_mapping.table_column_name = "a"; + child_mapping.file_type = nullable_int_type; + child_mapping.table_type = bigint_type; + child_mapping.is_trivial = false; + + ColumnMapping struct_mapping; + struct_mapping.file_type = file_struct_type; + struct_mapping.table_type = table_struct_type; + struct_mapping.child_mappings = {child_mapping}; + + auto child_values = ColumnInt32::create(); + child_values->insert_value(0); + child_values->insert_value(10); + auto child_null_map = ColumnUInt8::create(); + child_null_map->get_data().assign({1, 0}); + MutableColumns file_children; + file_children.push_back( + ColumnNullable::create(std::move(child_values), std::move(child_null_map))); + auto parent_null_map = ColumnUInt8::create(); + parent_null_map->get_data().assign({1, 0}); + ColumnPtr file_column = ColumnNullable::create(ColumnStruct::create(std::move(file_children)), + std::move(parent_null_map)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + const auto status = reader._materialize_struct_mapping_column(struct_mapping, file_column, 2, + &result_column); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& result_parent = assert_cast(*result_column); + const auto& result_struct = assert_cast(result_parent.get_nested_column()); + ASSERT_FALSE(result_struct.get_column(0).is_nullable()); + EXPECT_TRUE(result_parent.is_null_at(0)); + EXPECT_FALSE(result_parent.is_null_at(1)); + EXPECT_EQ(assert_cast(result_struct.get_column(0)).get_element(1), 10); +} + +TEST(TableReaderTest, ComplexRematerializeRejectsRequiredChildNullUnderPresentParent) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto file_struct_type = make_nullable( + std::make_shared(DataTypes {nullable_int_type}, Strings {"a"})); + const auto table_struct_type = + make_nullable(std::make_shared(DataTypes {bigint_type}, Strings {"a"})); + + ColumnMapping child_mapping; + child_mapping.file_local_id = 0; + child_mapping.file_column_name = "struct_column.a"; + child_mapping.table_column_name = "a"; + child_mapping.file_type = nullable_int_type; + child_mapping.table_type = bigint_type; + child_mapping.is_trivial = false; + + ColumnMapping struct_mapping; + struct_mapping.file_type = file_struct_type; + struct_mapping.table_type = table_struct_type; + struct_mapping.child_mappings = {child_mapping}; + + auto child_values = ColumnInt32::create(); + child_values->insert_value(0); + child_values->insert_value(0); + auto child_null_map = ColumnUInt8::create(); + child_null_map->get_data().assign({1, 1}); + MutableColumns file_children; + file_children.push_back( + ColumnNullable::create(std::move(child_values), std::move(child_null_map))); + auto parent_null_map = ColumnUInt8::create(); + parent_null_map->get_data().assign({1, 0}); + ColumnPtr file_column = ColumnNullable::create(ColumnStruct::create(std::move(file_children)), + std::move(parent_null_map)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + const auto status = reader._materialize_struct_mapping_column(struct_mapping, file_column, 2, + &result_column); + ASSERT_FALSE(status.ok()); +} + +TEST(TableReaderTest, ComplexRematerializeRejectsNullableFileStructForRequiredTableStruct) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto file_struct_type = make_nullable( + std::make_shared(DataTypes {nullable_int_type}, Strings {"a"})); + const auto table_struct_type = + std::make_shared(DataTypes {bigint_type}, Strings {"a"}); + + ColumnMapping child_mapping; + child_mapping.file_local_id = 0; + child_mapping.file_column_name = "struct_column.a"; + child_mapping.table_column_name = "a"; + child_mapping.file_type = nullable_int_type; + child_mapping.table_type = bigint_type; + child_mapping.is_trivial = false; + + ColumnMapping struct_mapping; + struct_mapping.file_type = file_struct_type; + struct_mapping.table_type = table_struct_type; + struct_mapping.child_mappings = {child_mapping}; + + auto child_values = ColumnInt32::create(); + child_values->insert_value(0); + child_values->insert_value(10); + auto child_null_map = ColumnUInt8::create(); + child_null_map->get_data().assign({1, 0}); + MutableColumns file_children; + file_children.push_back( + ColumnNullable::create(std::move(child_values), std::move(child_null_map))); + auto parent_null_map = ColumnUInt8::create(); + parent_null_map->get_data().assign({1, 0}); + ColumnPtr file_column = ColumnNullable::create(ColumnStruct::create(std::move(file_children)), + std::move(parent_null_map)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + const auto status = reader._materialize_struct_mapping_column(struct_mapping, file_column, 2, + &result_column); + ASSERT_FALSE(status.ok()); +} + +TEST(TableReaderTest, ComplexRematerializeAcceptsPresentFileStructForRequiredTableStruct) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto file_struct_type = make_nullable( + std::make_shared(DataTypes {nullable_int_type}, Strings {"a"})); + const auto table_struct_type = + std::make_shared(DataTypes {bigint_type}, Strings {"a"}); + + ColumnMapping child_mapping; + child_mapping.file_local_id = 0; + child_mapping.file_column_name = "struct_column.a"; + child_mapping.table_column_name = "a"; + child_mapping.file_type = nullable_int_type; + child_mapping.table_type = bigint_type; + child_mapping.is_trivial = false; + + ColumnMapping struct_mapping; + struct_mapping.file_type = file_struct_type; + struct_mapping.table_type = table_struct_type; + struct_mapping.child_mappings = {child_mapping}; + + auto child_values = ColumnInt32::create(); + child_values->insert_value(10); + child_values->insert_value(20); + MutableColumns file_children; + file_children.push_back( + ColumnNullable::create(std::move(child_values), ColumnUInt8::create(2, 0))); + ColumnPtr file_column = ColumnNullable::create(ColumnStruct::create(std::move(file_children)), + ColumnUInt8::create(2, 0)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + const auto status = reader._materialize_struct_mapping_column(struct_mapping, file_column, 2, + &result_column); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& result_struct = assert_cast(*result_column); + const auto& result_values = assert_cast(result_struct.get_column(0)); + EXPECT_EQ(result_values.get_element(0), 10); + EXPECT_EQ(result_values.get_element(1), 20); +} + +TEST(TableReaderTest, ComplexRematerializeCarriesAncestorMaskThroughRequiredStruct) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto file_inner_type = make_nullable( + std::make_shared(DataTypes {nullable_int_type}, Strings {"value"})); + const auto table_inner_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto file_outer_type = make_nullable( + std::make_shared(DataTypes {file_inner_type}, Strings {"inner"})); + const auto table_outer_type = make_nullable( + std::make_shared(DataTypes {table_inner_type}, Strings {"inner"})); + + ColumnMapping value_mapping; + value_mapping.file_local_id = 0; + value_mapping.file_column_name = "outer.inner.value"; + value_mapping.table_column_name = "value"; + value_mapping.file_type = nullable_int_type; + value_mapping.table_type = int_type; + value_mapping.is_trivial = true; + ColumnMapping inner_mapping; + inner_mapping.file_local_id = 0; + inner_mapping.file_column_name = "outer.inner"; + inner_mapping.table_column_name = "inner"; + inner_mapping.file_type = file_inner_type; + inner_mapping.table_type = table_inner_type; + inner_mapping.child_mappings = {value_mapping}; + ColumnMapping outer_mapping; + outer_mapping.file_type = file_outer_type; + outer_mapping.table_type = table_outer_type; + outer_mapping.child_mappings = {inner_mapping}; + + auto values = ColumnInt32::create(); + values->get_data().assign({0, 7}); + MutableColumns inner_children; + auto value_null_map = ColumnUInt8::create(); + value_null_map->get_data().assign({1, 0}); + inner_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto inner_null_map = ColumnUInt8::create(); + inner_null_map->get_data().assign({1, 0}); + auto inner = ColumnNullable::create(ColumnStruct::create(std::move(inner_children)), + std::move(inner_null_map)); + MutableColumns outer_children; + outer_children.push_back(std::move(inner)); + auto outer_null_map = ColumnUInt8::create(); + outer_null_map->get_data().assign({1, 0}); + ColumnPtr file_column = ColumnNullable::create(ColumnStruct::create(std::move(outer_children)), + std::move(outer_null_map)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + const auto status = reader._materialize_struct_mapping_column(outer_mapping, file_column, 2, + &result_column); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& result_outer = assert_cast(*result_column); + const auto& result_outer_struct = + assert_cast(result_outer.get_nested_column()); + const auto& result_inner = assert_cast(result_outer_struct.get_column(0)); + EXPECT_EQ(assert_cast(result_inner.get_column(0)).get_element(1), 7); +} + +TEST(TableReaderTest, ComplexRematerializeValidatesRequiredCollectionRootNulls) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto file_array_type = make_nullable(std::make_shared(nullable_int_type)); + const auto table_array_type = std::make_shared(nullable_int_type); + ColumnMapping element_mapping; + element_mapping.file_local_id = 0; + element_mapping.file_type = nullable_int_type; + element_mapping.table_type = int_type; + element_mapping.is_trivial = true; + ColumnMapping array_mapping; + array_mapping.file_type = file_array_type; + array_mapping.table_type = table_array_type; + array_mapping.child_mappings = {element_mapping}; + + auto nested_values = ColumnInt32::create(); + nested_values->get_data().assign({1, 1}); + auto values = ColumnNullable::create(std::move(nested_values), ColumnUInt8::create(2, 0)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().assign({1, 2}); + auto source_null_map = ColumnUInt8::create(); + source_null_map->get_data().assign({1, 0}); + ColumnPtr file_column = ColumnNullable::create( + ColumnArray::create(std::move(values), std::move(offsets)), std::move(source_null_map)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + EXPECT_FALSE( + reader._materialize_array_mapping_column(array_mapping, file_column, 2, &result_column) + .ok()); + NullMap ancestor_null_map(2, 0); + ancestor_null_map[0] = 1; + const auto status = reader._materialize_array_mapping_column( + array_mapping, file_column, 2, &result_column, &ancestor_null_map); + EXPECT_TRUE(status.ok()) << status.to_string(); +} + +TEST(TableReaderTest, ComplexRematerializeMasksArrayEntriesHiddenByNullRow) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto file_element_type = make_nullable( + std::make_shared(DataTypes {nullable_int_type}, Strings {"value"})); + const auto table_element_type = + std::make_shared(DataTypes {bigint_type}, Strings {"value"}); + const auto file_array_type = make_nullable(std::make_shared(file_element_type)); + const auto table_array_type = + make_nullable(std::make_shared(table_element_type)); + + ColumnMapping value_mapping; + value_mapping.table_column_name = "value"; + value_mapping.file_local_id = 0; + value_mapping.file_type = nullable_int_type; + value_mapping.table_type = bigint_type; + value_mapping.is_trivial = false; + ColumnMapping element_mapping; + element_mapping.file_local_id = 0; + element_mapping.file_type = file_element_type; + element_mapping.table_type = table_element_type; + element_mapping.child_mappings = {value_mapping}; + ColumnMapping array_mapping; + array_mapping.file_type = file_array_type; + array_mapping.table_type = table_array_type; + array_mapping.child_mappings = {element_mapping}; + + auto values = ColumnInt32::create(); + values->get_data().assign({0, 7}); + auto value_null_map = ColumnUInt8::create(); + value_null_map->get_data().assign({1, 0}); + MutableColumns element_children; + element_children.push_back( + ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto elements = ColumnNullable::create(ColumnStruct::create(std::move(element_children)), + ColumnUInt8::create(2, 0)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().assign({1, 2}); + auto array_null_map = ColumnUInt8::create(); + array_null_map->get_data().assign({1, 0}); + ColumnPtr file_column = + ColumnNullable::create(ColumnArray::create(std::move(elements), std::move(offsets)), + std::move(array_null_map)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + const auto status = + reader._materialize_array_mapping_column(array_mapping, file_column, 2, &result_column); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(assert_cast(*result_column).is_null_at(0)); +} + +TEST(TableReaderTest, ComplexRematerializeMasksMapEntriesHiddenByNullRow) { + const auto int_type = std::make_shared(); + const auto bigint_type = std::make_shared(); + const auto string_type = make_nullable(std::make_shared()); + const auto nullable_int_type = make_nullable(int_type); + const auto file_value_type = make_nullable( + std::make_shared(DataTypes {nullable_int_type}, Strings {"value"})); + const auto table_value_type = make_nullable( + std::make_shared(DataTypes {bigint_type}, Strings {"value"})); + const auto file_map_type = + make_nullable(std::make_shared(string_type, file_value_type)); + const auto table_map_type = + make_nullable(std::make_shared(string_type, table_value_type)); + + ColumnMapping key_mapping; + key_mapping.file_local_id = 0; + key_mapping.file_type = string_type; + key_mapping.table_type = string_type; + key_mapping.is_trivial = true; + ColumnMapping nested_value_mapping; + nested_value_mapping.table_column_name = "value"; + nested_value_mapping.file_local_id = 0; + nested_value_mapping.file_type = nullable_int_type; + nested_value_mapping.table_type = bigint_type; + nested_value_mapping.is_trivial = false; + ColumnMapping value_mapping; + value_mapping.file_local_id = 1; + value_mapping.file_type = file_value_type; + value_mapping.table_type = table_value_type; + value_mapping.child_mappings = {nested_value_mapping}; + ColumnMapping map_mapping; + map_mapping.file_type = file_map_type; + map_mapping.table_type = table_map_type; + map_mapping.child_mappings = {key_mapping, value_mapping}; + + auto keys = ColumnString::create(); + keys->insert_data("hidden", 6); + keys->insert_data("visible", 7); + auto values = ColumnInt32::create(); + values->get_data().assign({0, 9}); + auto value_null_map = ColumnUInt8::create(); + value_null_map->get_data().assign({1, 0}); + MutableColumns value_children; + value_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto map_values = ColumnNullable::create(ColumnStruct::create(std::move(value_children)), + ColumnUInt8::create(2, 0)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().assign({1, 2}); + auto map_null_map = ColumnUInt8::create(); + map_null_map->get_data().assign({1, 0}); + ColumnPtr file_column = ColumnNullable::create( + ColumnMap::create(ColumnNullable::create(std::move(keys), ColumnUInt8::create(2, 0)), + std::move(map_values), std::move(offsets)), + std::move(map_null_map)); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReaderCastTestHelper reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ColumnPtr result_column; + const auto status = + reader._materialize_map_mapping_column(map_mapping, file_column, 2, &result_column); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(assert_cast(*result_column).is_null_at(0)); +} + TEST(TableReaderTest, ReopenSplitAfterClose) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_test"; std::filesystem::remove_all(test_dir); @@ -4389,6 +4913,74 @@ TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { } } +TEST(TableReaderTest, ArrayElementMaterializationPreservesNullMap) { + const auto int_type = make_nullable(std::make_shared()); + const auto string_type = make_nullable(std::make_shared()); + const auto struct_type = std::make_shared(DataTypes {int_type, string_type}, + Strings {"i_info", "s_info"}); + const auto nullable_struct_type = make_nullable(struct_type); + const auto array_type = make_nullable(std::make_shared(nullable_struct_type)); + + auto table_column = make_table_column(0, "ss_info", array_type); + auto table_element = make_table_column(0, "element", struct_type); + table_element.type = struct_type; + table_column.children = {table_element}; + + auto file_column = make_file_column(0, "ss_info", array_type); + auto file_element = make_file_column(0, "element", nullable_struct_type); + file_element.children = { + make_file_column(0, "i_info", int_type), + make_file_column(1, "s_info", string_type), + }; + file_column.children = {file_element}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, {file_column}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + + auto int_values = ColumnInt32::create(); + int_values->get_data().assign({0, 0, 5}); + auto string_values = ColumnString::create(); + string_values->insert_default(); + string_values->insert_default(); + string_values->insert_data("doris-nereids-5", 15); + MutableColumns struct_children; + struct_children.push_back( + ColumnNullable::create(std::move(int_values), ColumnUInt8::create(3, 0))); + struct_children.push_back( + ColumnNullable::create(std::move(string_values), ColumnUInt8::create(3, 0))); + auto element_null_map = ColumnUInt8::create(); + element_null_map->get_data().assign({1, 1, 0}); + auto elements = ColumnNullable::create(ColumnStruct::create(std::move(struct_children)), + std::move(element_null_map)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(3); + auto root_null_map = ColumnUInt8::create(1, 0); + ColumnPtr file_data = ColumnNullable::create( + ColumnArray::create(std::move(elements), std::move(offsets)), std::move(root_null_map)); + + TableReaderCastTestHelper reader; + ColumnPtr result; + ASSERT_TRUE( + reader._materialize_array_mapping_column(mapper.mappings()[0], file_data, 1, &result) + .ok()); + const auto& result_array = assert_cast( + assert_cast(*result).get_nested_column()); + const auto& result_elements = assert_cast(result_array.get_data()); + EXPECT_TRUE(result_elements.is_null_at(0)); + EXPECT_TRUE(result_elements.is_null_at(1)); + EXPECT_FALSE(result_elements.is_null_at(2)); + const auto& result_struct = + assert_cast(result_elements.get_nested_column()); + const auto& result_ints = assert_cast( + assert_cast(result_struct.get_column(0)).get_nested_column()); + EXPECT_EQ(result_ints.get_element(2), 5); + const auto& result_strings = assert_cast( + assert_cast(result_struct.get_column(1)).get_nested_column()); + EXPECT_EQ(result_strings.get_data_at(2).to_string(), "doris-nereids-5"); +} + TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) { const auto int_type = std::make_shared(); const std::vector projected_columns = { diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index bcc8b6aa496d62..fe82e229c8119d 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -454,6 +454,10 @@ public boolean typeContainsPrecision() { } public String hideVersionForVersionColumn(Boolean isToSql) { + return hideVersionForVersionColumn(isToSql, false); + } + + public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedComment) { if (isDatetime() || isDatetimeV2()) { StringBuilder typeStr = new StringBuilder("datetime"); if (((ScalarType) this).getScalarScale() > 0) { @@ -482,18 +486,32 @@ public String hideVersionForVersionColumn(Boolean isToSql) { } return typeStr.toString(); } else if (isArrayType()) { - String nestedDesc = ((ArrayType) this).getItemType().hideVersionForVersionColumn(isToSql); + String nestedDesc = ((ArrayType) this).getItemType() + .hideVersionForVersionColumn(isToSql, showNestedComment); return "array<" + nestedDesc + ">"; } else if (isMapType()) { - String keyDesc = ((MapType) this).getKeyType().hideVersionForVersionColumn(isToSql); - String valueDesc = ((MapType) this).getValueType().hideVersionForVersionColumn(isToSql); + String keyDesc = ((MapType) this).getKeyType() + .hideVersionForVersionColumn(isToSql, showNestedComment); + String valueDesc = ((MapType) this).getValueType() + .hideVersionForVersionColumn(isToSql, showNestedComment); return "map<" + keyDesc + "," + valueDesc + ">"; } else if (isStructType()) { List fieldDesc = new ArrayList<>(); StructType structType = (StructType) this; for (int i = 0; i < structType.getFields().size(); i++) { StructField field = structType.getFields().get(i); - fieldDesc.add(field.getName() + ":" + field.getType().hideVersionForVersionColumn(isToSql)); + StringBuilder desc = new StringBuilder(field.getName()).append(":") + .append(field.getType().hideVersionForVersionColumn(isToSql, showNestedComment)); + // Requiredness is schema semantics and must survive independently of whether + // nested documentation is requested for DESCRIBE output. + if (!field.getContainsNull()) { + desc.append(" not null"); + } + // Nested docs are part of DESCRIBE output only when comments were explicitly requested. + if (showNestedComment && field.isCommentSpecified()) { + desc.append(String.format(" comment '%s'", field.getComment())); + } + fieldDesc.add(desc.toString()); } return "struct<" + StringUtils.join(fieldDesc, ",") + ">"; } else if (isToSql) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index b32dd168ffcdef..7578685a771d64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -65,6 +65,8 @@ public static ProcResult createResult(List schema, Set bfColumns } } result.setNames(names); + boolean showNestedComment = additionalColNames.stream() + .anyMatch(name -> "comment".equalsIgnoreCase(name)); for (Column column : schema) { // Extra string (aggregation and bloom filter) @@ -87,7 +89,7 @@ public static ProcResult createResult(List schema, Set bfColumns String extraStr = StringUtils.join(extras, ","); List rowList = Lists.newArrayList(column.getDisplayName(), - column.getOriginType().hideVersionForVersionColumn(true), + column.getOriginType().hideVersionForVersionColumn(true, showNestedComment), column.isAllowNull() ? "Yes" : "No", ((Boolean) column.isKey()).toString(), column.getDefaultValue() == null diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index f5786423b6e9e0..d238ab9556dfc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -178,6 +178,10 @@ public List getFullSchema() { return schemaCacheValue.map(SchemaCacheValue::getSchema).orElse(null); } + public List getFullSchema(Optional snapshot) { + return getFullSchema(); + } + protected boolean needInternalHiddenColumns() { return false; } @@ -193,7 +197,11 @@ public List getBaseSchema() { @Override public List getBaseSchema(boolean full) { - List schema = getFullSchema(); + return getBaseSchema(Optional.empty(), full); + } + + public List getBaseSchema(Optional snapshot, boolean full) { + List schema = snapshot.isPresent() ? getFullSchema(snapshot) : getFullSchema(); if (schema == null) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 1cc71f863d27af..873ab200350297 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -38,6 +38,9 @@ import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.source.HiveSplit; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccTable; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.ConnectContext; @@ -106,6 +109,8 @@ public abstract class FileQueryScanNode extends FileScanNode { protected SessionVariable sessionVariable; protected TableScanParams scanParams; + private Optional relationSnapshot = Optional.empty(); + private boolean relationSnapshotInitialized = false; protected FileSplitter fileSplitter; protected SummaryProfile summaryProfile; @@ -182,7 +187,9 @@ protected void initSchemaParams() throws UserException { params = new TFileScanRangeParams(); params.setDestTupleId(desc.getId().asInt()); List partitionKeys = getPathPartitionKeys(); - List columns = desc.getTable().getBaseSchema(false); + List columns = desc.getTable() instanceof ExternalTable + ? ((ExternalTable) desc.getTable()).getBaseSchema(getRelationSnapshot(), false) + : desc.getTable().getBaseSchema(false); params.setNumOfColumnsFromFile(columns.size() - partitionKeys.size()); for (SlotDescriptor slot : desc.getSlots()) { TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); @@ -192,7 +199,12 @@ protected void initSchemaParams() throws UserException { slotInfo.setIsFileSlot(isFileSlot(category)); params.addToRequiredSlots(slotInfo); } - setDefaultValueExprs(getTargetTable(), destSlotDescByName, null, params, false); + // Defaults are field semantics, so resolve them from the same relation schema as the + // tuple slots; matching a newer same-named field could attach the wrong default. + List defaultValueColumns = desc.getTable() instanceof ExternalTable + ? ((ExternalTable) desc.getTable()).getFullSchema(getRelationSnapshot()) + : desc.getTable().getFullSchema(); + setDefaultValueExprs(getTargetTable(), destSlotDescByName, null, params, false, defaultValueColumns); setColumnPositionMapping(); // For query, set src tuple id to -1. params.setSrcTupleId(-1); @@ -308,7 +320,12 @@ private void setColumnPositionMapping() } protected List getFileColumnNames() { - return desc.getTable().getFullSchema().stream() + // Default positions must follow this relation's snapshot when one statement scans + // multiple versions; format-specific subclasses may instead expose a processed schema. + List columns = desc.getTable() instanceof ExternalTable + ? ((ExternalTable) desc.getTable()).getFullSchema(getRelationSnapshot()) + : desc.getTable().getFullSchema(); + return columns.stream() .map(Column::getName) .collect(Collectors.toList()); } @@ -597,7 +614,10 @@ private TFileRangeDesc createFileRangeDesc(FileSplit fileSplit, List col // We need to save mapping from slot name to schema position protected void genSlotToSchemaIdMapForOrc() { Preconditions.checkNotNull(params); - List baseSchema = desc.getTable().getBaseSchema(); + // ORC positions are relation-local for the same reason as the regular column mapping. + List baseSchema = desc.getTable() instanceof ExternalTable + ? ((ExternalTable) desc.getTable()).getBaseSchema(getRelationSnapshot(), false) + : desc.getTable().getBaseSchema(); Map columnNameToPosition = Maps.newHashMap(); for (SlotDescriptor slot : desc.getSlots()) { int idx = 0; @@ -743,6 +763,35 @@ public TableScanParams getScanParams() { return this.scanParams; } + public void setRelationSnapshot(Optional relationSnapshot) { + // The resolved snapshot is part of the bound relation; reloading a movable ref here + // could make execution use different metadata from the schema used during analysis. + this.relationSnapshot = relationSnapshot; + this.relationSnapshotInitialized = true; + } + + /** + * Return metadata pinned for this scan relation. + */ + protected Optional getRelationSnapshot() { + if (relationSnapshotInitialized) { + return relationSnapshot; + } + relationSnapshotInitialized = true; + TableIf targetTable = desc.getTable(); + if (!(targetTable instanceof MvccTable)) { + return Optional.empty(); + } + if (tableSnapshot != null || scanParams != null) { + // Legacy planner callers do not carry a bound snapshot, so resolve their qualifiers here. + relationSnapshot = Optional.of(((MvccTable) targetTable).loadSnapshot( + Optional.ofNullable(tableSnapshot), Optional.ofNullable(scanParams))); + return relationSnapshot; + } + relationSnapshot = MvccUtil.getSnapshotFromContext(targetTable); + return relationSnapshot; + } + protected boolean fileCacheAdmissionCheck() throws UserException { boolean admissionResultAtTableLevel = true; TableIf tableIf = getTargetTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index f017c444c69122..0469c0ce1e389d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -42,6 +42,7 @@ import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanNode; import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TFileScanSlotInfo; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TPushAggOp; @@ -271,6 +272,16 @@ protected void setDefaultValueExprs(TableIf tbl, Map exprByName, TFileScanRangeParams params, boolean useVarcharAsNull) throws UserException { + setDefaultValueExprs(tbl, slotDescByName, exprByName, params, useVarcharAsNull, + desc.getTable().getFullSchema()); + } + + protected void setDefaultValueExprs(TableIf tbl, + Map slotDescByName, + Map exprByName, + TFileScanRangeParams params, + boolean useVarcharAsNull, + List columns) throws UserException { Preconditions.checkNotNull(tbl); TExpr tExpr = new TExpr(); tExpr.setNodes(Lists.newArrayList()); @@ -280,7 +291,16 @@ protected void setDefaultValueExprs(TableIf tbl, nameToSlotDesc.put(slot.getColumn().getName(), slot); } - for (Column column : desc.getTable().getFullSchema()) { + // Build slot_id -> index map for required_slots to set default_value_expr inline. + Map slotIdToRequiredIdx = Maps.newHashMap(); + if (params.getRequiredSlots() != null) { + for (int i = 0; i < params.getRequiredSlots().size(); i++) { + TFileScanSlotInfo slotInfo = params.getRequiredSlots().get(i); + slotIdToRequiredIdx.put(slotInfo.getSlotId(), i); + } + } + + for (Column column : columns) { Expr expr; Expression expression; if (column.getDefaultValue() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 03a37937562952..a727d4386f774a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -38,6 +38,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.hudi.HudiExternalMetaCache; +import org.apache.doris.datasource.hudi.HudiMvccSnapshot; import org.apache.doris.datasource.hudi.HudiSchemaCacheKey; import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; import org.apache.doris.datasource.hudi.HudiUtils; @@ -381,6 +382,19 @@ public List getFullSchema() { return schemaCacheValue.map(SchemaCacheValue::getSchema).orElse(null); } + @Override + public List getFullSchema(Optional snapshot) { + makeSureInitialized(); + // HMS can front snapshot-aware formats too; callers that pin a relation must not fall + // back to the table's current schema merely because its catalog type is HMS. + if (getDlaType() == DLAType.HUDI) { + return ((HudiDlaTable) dlaTable).getHudiSchemaCacheValue(snapshot).getSchema(); + } else if (getDlaType() == DLAType.ICEBERG) { + return IcebergUtils.getIcebergSchema(this, snapshot); + } + return super.getFullSchema(snapshot); + } + @Override public Optional getSchemaCacheValue() { makeSureInitialized(); @@ -462,23 +476,32 @@ public SelectedPartitions initHudiSelectedPartitions(Optional tab return SelectedPartitions.NOT_PRUNED; } TablePartitionValues tablePartitionValues = HudiUtils.getPartitionValues(tableSnapshot, this); - - Map idToPartitionItem = tablePartitionValues.getIdToPartitionItem(); - Map idToNameMap = tablePartitionValues.getPartitionIdToNameMap(); - - Map nameToPartitionItems = Maps.newHashMapWithExpectedSize(idToPartitionItem.size()); - for (Entry entry : idToPartitionItem.entrySet()) { - nameToPartitionItems.put(idToNameMap.get(entry.getKey()), entry.getValue()); - } + Map nameToPartitionItems = toNameToPartitionItems(tablePartitionValues); return new SelectedPartitions(nameToPartitionItems.size(), nameToPartitionItems, false); } @Override public Map getNameToPartitionItems(Optional snapshot) { + if (getDlaType() == DLAType.HUDI && snapshot.filter(HudiMvccSnapshot.class::isInstance).isPresent()) { + // Hudi binding already resolved the timeline; reuse its frozen partition values so + // schema, partition pruning, and split planning stay on one metadata generation. + HudiMvccSnapshot hudiSnapshot = (HudiMvccSnapshot) snapshot.get(); + return toNameToPartitionItems(hudiSnapshot.getTablePartitionValues()); + } return getNameToPartitionItems(); } + private Map toNameToPartitionItems(TablePartitionValues tablePartitionValues) { + Map idToPartitionItem = tablePartitionValues.getIdToPartitionItem(); + Map idToNameMap = tablePartitionValues.getPartitionIdToNameMap(); + Map nameToPartitionItems = Maps.newHashMapWithExpectedSize(idToPartitionItem.size()); + for (Entry entry : idToPartitionItem.entrySet()) { + nameToPartitionItems.put(idToNameMap.get(entry.getKey()), entry.getValue()); + } + return nameToPartitionItems; + } + public Map getNameToPartitionItems() { if (CollectionUtils.isEmpty(this.getPartitionColumns())) { return Collections.emptyMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiMvccSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiMvccSnapshot.java index 8821d113a7f5ea..20ed6ce9009ef4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiMvccSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiMvccSnapshot.java @@ -56,6 +56,11 @@ public TablePartitionValues getTablePartitionValues() { return tablePartitionValues; } + @Override + public boolean isSameSnapshot(MvccSnapshot other) { + return other instanceof HudiMvccSnapshot && timestamp == ((HudiMvccSnapshot) other).timestamp; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -65,12 +70,12 @@ public boolean equals(Object o) { return false; } HudiMvccSnapshot that = (HudiMvccSnapshot) o; - return tablePartitionValues.equals(that.tablePartitionValues); + return timestamp == that.timestamp && tablePartitionValues.equals(that.tablePartitionValues); } @Override public int hashCode() { - return tablePartitionValues.hashCode(); + return 31 * tablePartitionValues.hashCode() + Long.hashCode(timestamp); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java index 6c622e371c319e..2e53c69c1e1ec3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java @@ -32,6 +32,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.hive.HivePartition; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TArrayField; @@ -265,7 +266,46 @@ public static HudiMvccSnapshot getHudiMvccSnapshot(Optional table timestamp = getLastTimeStamp(hmsTable); } - return new HudiMvccSnapshot(HudiUtils.getPartitionValues(tableSnapshot, hmsTable), timestamp); + return new HudiMvccSnapshot(getPartitionValuesAtInstant(tableSnapshot, hmsTable, timestamp), timestamp); + } + + private static TablePartitionValues getPartitionValuesAtInstant(Optional tableSnapshot, + HMSExternalTable hmsTable, long timestamp) { + if (timestamp <= 0 || tableSnapshot.filter( + snapshot -> snapshot.getType() == TableSnapshot.VersionType.VERSION).isPresent()) { + return new TablePartitionValues(); + } + + HudiExternalMetaCache hudiExternalMetaCache = Env.getCurrentEnv().getExtMetaCacheMgr() + .hudi(hmsTable.getCatalog().getId()); + try { + // Partition metadata must use the same instant captured above; a latest-cache lookup + // could cross a refresh boundary and combine partitions from a newer generation. + return hmsTable.getCatalog().getExecutionAuthenticator().execute(() -> + hudiExternalMetaCache.getSnapshotPartitionValues( + hmsTable, Long.toString(timestamp), hmsTable.useHiveSyncPartition())); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } + + /** + * Resolve the instant used for split and schema planning. + */ + public static Optional resolveQueryInstant(Optional relationSnapshot, + Optional tableSnapshot, HoodieTimeline timeline) { + if (relationSnapshot.filter(HudiMvccSnapshot.class::isInstance).isPresent()) { + long timestamp = ((HudiMvccSnapshot) relationSnapshot.get()).getTimestamp(); + // The pinned timestamp is authoritative even if the active timeline advances later. + return timestamp > 0 ? Optional.of(Long.toString(timestamp)) : Optional.empty(); + } + if (tableSnapshot.isPresent()) { + return Optional.of(tableSnapshot.get().getValue().replaceAll("[-: ]", "")); + } + Option snapshotInstant = timeline.lastInstant(); + return snapshotInstant.isPresent() + ? Optional.of(snapshotInstant.get().requestedTime()) + : Optional.empty(); } public static long getLastTimeStamp(HMSExternalTable hmsTable) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index d5b454a8a5023c..91c6da61a5747c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -38,7 +38,7 @@ import org.apache.doris.datasource.hudi.HudiPartitionUtils; import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; import org.apache.doris.datasource.hudi.HudiUtils; -import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -61,7 +61,6 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.TableSchemaResolver; -import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.Option; @@ -163,6 +162,7 @@ public TFileFormatType getFileFormatType() throws UserException { @Override protected void doInitialize() throws UserException { ExternalTable table = (ExternalTable) desc.getTable(); + Optional relationSnapshot = getRelationSnapshot(); if (table.isView()) { throw new AnalysisException( String.format("Querying external view '%s.%s' is not supported", table.getDbName(), @@ -205,20 +205,17 @@ protected void doInitialize() throws UserException { timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); TableSnapshot tableSnapshot = getQueryTableSnapshot(); - if (tableSnapshot != null) { - if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) { - throw new UserException("Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"); - } - queryInstant = tableSnapshot.getValue().replaceAll("[-: ]", ""); - } else { - Option snapshotInstant = timeline.lastInstant(); - if (!snapshotInstant.isPresent()) { - prunedPartitions = Collections.emptyList(); - partitionInit = true; - return; - } - queryInstant = snapshotInstant.get().requestedTime(); + if (tableSnapshot != null && tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) { + throw new UserException("Hudi does not support `FOR VERSION AS OF`, please use `FOR TIME AS OF`"); + } + Optional resolvedInstant = HudiUtils.resolveQueryInstant( + relationSnapshot, Optional.ofNullable(tableSnapshot), timeline); + if (!resolvedInstant.isPresent()) { + prunedPartitions = Collections.emptyList(); + partitionInit = true; + return; } + queryInstant = resolvedInstant.get(); HudiSchemaCacheValue hudiSchemaCacheValue = HudiUtils.getSchemaCacheValue(hmsTable, queryInstant); columnNames = hudiSchemaCacheValue.getSchema().stream().map(Column::getName).collect(Collectors.toList()); @@ -238,7 +235,8 @@ protected void doInitialize() throws UserException { // `table_info_node_ptr` will be `TableSchemaChangeHelper::ConstNode`. When using `ConstNode`, // you need to pay special attention to the `case difference` between the `table column name` // and `the file column name`. - ExternalUtil.initSchemaInfo(params, -1L, table.getColumns()); + // Split planning and FE-BE schema transport must describe the same pinned Hudi instant. + ExternalUtil.initSchemaInfo(params, -1L, table.getFullSchema(relationSnapshot)); } @Override @@ -344,7 +342,7 @@ private boolean canUseNativeReader() { private List getPrunedPartitions(HoodieTableMetaClient metaClient) { NameMapping nameMapping = hmsTable.getOrBuildNameMapping(); - List partitionColumnTypes = hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable)); + List partitionColumnTypes = hmsTable.getPartitionColumnTypes(getRelationSnapshot()); if (!partitionColumnTypes.isEmpty()) { this.totalPartitionNum = selectedPartitions.totalPartitionNum; Map prunedPartitions = selectedPartitions.selectedPartitions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 047272916fbee6..8407a29d8908a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -222,17 +222,21 @@ private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTabl dorisTable.getDbName(), dorisTable.getName())); } try { + // Freeze before deriving snapshot, partitions, and aliases; BaseTable accessors share + // refreshable operations and otherwise could mix two concurrent metadata generations. + Table retainedTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); MTMVRelatedTableIf table = (MTMVRelatedTableIf) dorisTable; - IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(icebergTable); + IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(retainedTable); IcebergPartitionInfo icebergPartitionInfo; if (!table.isValidRelatedTable()) { icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { - icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, + icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, retainedTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } return new IcebergSnapshotCacheValue( - icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(icebergTable)); + icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(retainedTable), + retainedTable); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 7dfdf6aed929bb..5d59440a5a62b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -291,14 +291,25 @@ protected boolean needInternalHiddenColumns() { @Override public List getFullSchema() { - List schema = IcebergUtils.getIcebergSchema(this); + return getFullSchema(MvccUtil.getSnapshotFromContext(this)); + } + + @Override + public List getFullSchema(Optional snapshot) { + List schema = IcebergUtils.getIcebergSchema(this, snapshot); schema = new ArrayList<>(schema); if (Util.showHiddenColumns() || needInternalHiddenColumns()) { schema.add(createIcebergRowIdColumn()); } - schema = IcebergUtils.appendRowLineageColumnsForV3(schema, getIcebergTable()); + Optional snapshotTable = snapshot + .filter(IcebergMvccSnapshot.class::isInstance) + .map(IcebergMvccSnapshot.class::cast) + .flatMap(value -> value.getSnapshotCacheValue().getIcebergTable()); + // Row-lineage fields are part of the pinned schema generation, not the refreshable table. + schema = IcebergUtils.appendRowLineageColumnsForV3( + schema, snapshotTable.orElseGet(this::getIcebergTable)); return schema; } @@ -441,7 +452,11 @@ public boolean isPartitionedTable() { * @return SQL string representing ORDER BY clause, or empty string if no sort order */ public String getSortOrderSql() { - Table table = getIcebergTable(); + return getSortOrderSql(getIcebergTable()); + } + + /** Return the sort order SQL for an already resolved Iceberg metadata generation. */ + public String getSortOrderSql(Table table) { org.apache.iceberg.SortOrder sortOrder = table.sortOrder(); if (sortOrder == null || sortOrder.isUnsorted() || sortOrder.fields().isEmpty()) { return ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMvccSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMvccSnapshot.java index 2c0155a71cd389..4ffd1d908c8927 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMvccSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMvccSnapshot.java @@ -29,4 +29,19 @@ public IcebergMvccSnapshot(IcebergSnapshotCacheValue snapshotCacheValue) { public IcebergSnapshotCacheValue getSnapshotCacheValue() { return snapshotCacheValue; } + + @Override + public boolean isSameSnapshot(MvccSnapshot other) { + if (!(other instanceof IcebergMvccSnapshot)) { + return false; + } + IcebergSnapshot left = snapshotCacheValue.getSnapshot(); + IcebergSnapshotCacheValue otherCacheValue = ((IcebergMvccSnapshot) other).snapshotCacheValue; + IcebergSnapshot right = otherCacheValue.getSnapshot(); + // A branch can retain its data snapshot while adopting a newer current schema. + // Name mapping is also scan-visible state and may change without a snapshot or schema ID change. + return left.getSnapshotId() == right.getSnapshotId() + && left.getSchemaId() == right.getSchemaId() + && snapshotCacheValue.getNameMapping().equals(otherCacheValue.getNameMapping()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 0c78be1accf76c..30cf64fcfc6bfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,11 +17,22 @@ package org.apache.doris.datasource.iceberg; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.LocationProvider; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; public class IcebergSnapshotCacheValue { @@ -29,15 +40,29 @@ public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; private final Optional>> nameMapping; + private final Optional
icebergTable; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { - this(partitionInfo, snapshot, Optional.empty()); + this(partitionInfo, snapshot, Optional.empty(), Optional.empty()); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping) { + this(partitionInfo, snapshot, nameMapping, Optional.empty()); + } + + public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping, Table icebergTable) { + this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable)); + } + + private IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping, Optional
icebergTable) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; + // A cached BaseTable shares live TableOperations; retain a metadata-only generation so a + // later commit through that same Table cannot move an already bound statement forward. + this.icebergTable = icebergTable.map(IcebergSnapshotCacheValue::retainTableGeneration); this.nameMapping = nameMapping.map(mapping -> { Map> copy = new HashMap<>(); // Preserve the immutable snapshot contract while remaining compatible with branch-4.1's Java target. @@ -58,4 +83,142 @@ public IcebergSnapshot getSnapshot() { public Optional>> getNameMapping() { return nameMapping; } + + public Optional
getIcebergTable() { + return icebergTable; + } + + static Table retainTableGeneration(Table table) { + if (!(table instanceof HasTableOperations) || isFrozenGeneration(table)) { + return table; + } + TableOperations operations = ((HasTableOperations) table).operations(); + // Capture current() exactly once so every projection derived from the returned table sees + // one metadata generation even when the shared catalog handle refreshes concurrently. + TableOperations frozenOperations = new FrozenTableOperations(operations, operations.current()); + return tableWithOperations(table, frozenOperations); + } + + static boolean isFrozenGeneration(Table table) { + return table instanceof HasTableOperations + && ((HasTableOperations) table).operations() instanceof FrozenTableOperations; + } + + static Table createWritableTable(Table retainedTable, Table liveTable) { + if (!isFrozenGeneration(retainedTable)) { + return retainedTable; + } + if (!(liveTable instanceof HasTableOperations) + || isFrozenGeneration(liveTable)) { + throw new IllegalArgumentException( + "Iceberg commit table must provide writable table operations"); + } + TableMetadata retainedMetadata = ((HasTableOperations) retainedTable).operations().current(); + TableOperations liveOperations = ((HasTableOperations) liveTable).operations(); + return tableWithOperations(retainedTable, + new WritableTableOperations(liveOperations, retainedMetadata)); + } + + private static Table tableWithOperations(Table table, TableOperations operations) { + if (table instanceof BaseTable) { + return new BaseTable(operations, table.name(), ((BaseTable) table).reporter()); + } + return new BaseTable(operations, table.name()); + } + + private abstract static class RetainedTableOperations implements TableOperations { + protected final TableOperations delegate; + private final TableMetadata metadata; + + private RetainedTableOperations(TableOperations delegate, TableMetadata metadata) { + this.delegate = delegate; + this.metadata = metadata; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; + } + + @Override + public FileIO io() { + return delegate.io(); + } + + @Override + public EncryptionManager encryption() { + return delegate.encryption(); + } + + @Override + public String metadataFileLocation(String fileName) { + return delegate.metadataFileLocation(fileName); + } + + @Override + public LocationProvider locationProvider() { + return delegate.locationProvider(); + } + } + + private static class FrozenTableOperations extends RetainedTableOperations { + private FrozenTableOperations(TableOperations delegate, TableMetadata metadata) { + super(delegate, metadata); + } + + @Override + public void commit(TableMetadata base, TableMetadata newMetadata) { + throw new UnsupportedOperationException("Frozen Iceberg table generation is read-only"); + } + } + + private static class WritableTableOperations extends RetainedTableOperations { + private final TableMetadata retainedMetadata; + private TableMetadata currentMetadata; + + private WritableTableOperations(TableOperations delegate, TableMetadata retainedMetadata) { + super(delegate, retainedMetadata); + this.retainedMetadata = retainedMetadata; + this.currentMetadata = retainedMetadata; + } + + @Override + public TableMetadata current() { + return currentMetadata; + } + + @Override + public TableMetadata refresh() { + TableMetadata refreshedMetadata = delegate.refresh(); + // Data-only snapshot advances are safe to replay, but a changed writer contract must + // fail instead of silently committing files produced for another metadata generation. + if (!isWriterCompatible(refreshedMetadata)) { + throw new CommitFailedException( + "Cannot retry Iceberg commit after schema, spec, sort order, location, " + + "format version, or table properties changed"); + } + currentMetadata = refreshedMetadata; + return refreshedMetadata; + } + + @Override + public void commit(TableMetadata base, TableMetadata newMetadata) { + delegate.commit(base, newMetadata); + currentMetadata = newMetadata; + } + + private boolean isWriterCompatible(TableMetadata refreshedMetadata) { + return retainedMetadata.formatVersion() == refreshedMetadata.formatVersion() + && retainedMetadata.currentSchemaId() == refreshedMetadata.currentSchemaId() + && retainedMetadata.defaultSpecId() == refreshedMetadata.defaultSpecId() + && retainedMetadata.defaultSortOrderId() == refreshedMetadata.defaultSortOrderId() + && Objects.equals(retainedMetadata.location(), refreshedMetadata.location()) + && Objects.equals(retainedMetadata.properties(), refreshedMetadata.properties()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index aeb539d1f3fa3d..28e45b47acd250 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -80,6 +80,25 @@ public String getSysTableType() { return sysTableType; } + public boolean isPositionDeletesTable() { + return MetadataTableType.POSITION_DELETES.name().equalsIgnoreCase(sysTableType); + } + + public boolean supportsSnapshotSelection() { + MetadataTableType tableType = MetadataTableType.from(sysTableType); + // Static metadata scans and ALL_* scans ignore a selected snapshot, so accepting one would + // only add failures for expired IDs without changing the rows returned. + return tableType != MetadataTableType.ALL_DATA_FILES + && tableType != MetadataTableType.ALL_DELETE_FILES + && tableType != MetadataTableType.ALL_FILES + && tableType != MetadataTableType.ALL_MANIFESTS + && tableType != MetadataTableType.ALL_ENTRIES + && tableType != MetadataTableType.HISTORY + && tableType != MetadataTableType.SNAPSHOTS + && tableType != MetadataTableType.REFS + && tableType != MetadataTableType.METADATA_LOG_ENTRIES; + } + public Table getSysIcebergTable() { if (sysIcebergTable == null) { synchronized (this) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 1325df321c37ee..71935cbd88157b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -61,6 +61,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -126,12 +127,15 @@ public void updateRewriteFiles(List filesToDelete) { } } - public void beginInsert(ExternalTable dorisTable, Optional ctx) throws UserException { + /** Begin an insert against the metadata generation retained during sink binding. */ + public void beginInsert(ExternalTable dorisTable, Table targetTable, + Optional ctx) throws UserException { ctx.ifPresent(c -> this.insertCtx = (IcebergInsertCommandContext) c); try { ops.getExecutionAuthenticator().execute(() -> { - // create and start the iceberg transaction - this.table = IcebergUtils.getIcebergTable(dorisTable); + // Planning, BE serialization, and commit must share one Iceberg metadata + // generation even if the catalog refreshes between those phases. + this.table = targetTable; this.baseSnapshotId = null; // check branch if (insertCtx != null && insertCtx.getBranchName().isPresent()) { @@ -145,28 +149,26 @@ public void beginInsert(ExternalTable dorisTable, Optional + " is a tag, not a branch. Tags cannot be targets for producing snapshots"); } } - this.transaction = table.newTransaction(); + this.transaction = createTransactionTable(dorisTable, table).newTransaction(); this.rewrittenDeleteFilesByReferencedDataFile = Collections.emptyMap(); }); } catch (Exception e) { throw new UserException("Failed to begin insert for iceberg table " + dorisTable.getName() - + "because: " + e.getMessage(), e); + + " because: " + e.getMessage(), e); } } - /** - * Begin rewrite transaction for data file rewrite operations - */ - public void beginRewrite(ExternalTable dorisTable) throws UserException { + /** Begin a rewrite against the same retained table used by every rewrite task. */ + public void beginRewrite(ExternalTable dorisTable, Table targetTable) throws UserException { // For rewrite operations, we work directly on the main table this.branchName = null; this.isRewriteMode = true; try { ops.getExecutionAuthenticator().execute(() -> { - // create and start the iceberg transaction - this.table = IcebergUtils.getIcebergTable(dorisTable); + // A rewrite group must not silently switch to refreshed metadata mid-transaction. + this.table = targetTable; this.baseSnapshotId = null; // Capture the starting snapshot ID for validation during rewrite commit @@ -175,7 +177,7 @@ public void beginRewrite(ExternalTable dorisTable) throws UserException { // For rewrite operations, we work directly on the main table // No branch information needed - this.transaction = table.newTransaction(); + this.transaction = createTransactionTable(dorisTable, table).newTransaction(); LOG.info("Started rewrite transaction for table: {} (main table)", dorisTable.getName()); return null; @@ -265,11 +267,12 @@ private void updateManifestAfterRewrite() { /** * Begin delete operation for Iceberg table */ - public void beginDelete(ExternalTable dorisTable) throws UserException { + public void beginDelete(ExternalTable dorisTable, Table targetTable) throws UserException { try { ops.getExecutionAuthenticator().execute(() -> { - // create and start the iceberg transaction - this.table = IcebergUtils.getIcebergTable(dorisTable); + // RowDelta's validation base must match the generation used to select row IDs; + // reloading the live table here could silently include a concurrent commit. + this.table = targetTable; this.baseSnapshotId = getSnapshotIdIfPresent(table); if (table instanceof org.apache.iceberg.HasTableOperations) { int formatVersion = ((org.apache.iceberg.HasTableOperations) table).operations() @@ -279,7 +282,7 @@ public void beginDelete(ExternalTable dorisTable) throws UserException { + " must have format version 2 or higher for position deletes"); } } - this.transaction = table.newTransaction(); + this.transaction = createTransactionTable(dorisTable, table).newTransaction(); this.rewrittenDeleteFilesByReferencedDataFile = Collections.emptyMap(); LOG.info("Started delete transaction for table: {}", dorisTable.getName()); }); @@ -289,14 +292,23 @@ public void beginDelete(ExternalTable dorisTable) throws UserException { } } - /** - * Begin merge operation for Iceberg UPDATE (single scan RowDelta). - */ - public void beginMerge(ExternalTable dorisTable) throws UserException { + private Table createTransactionTable(ExternalTable dorisTable, Table retainedTable) { + if (!IcebergSnapshotCacheValue.isFrozenGeneration(retainedTable)) { + return retainedTable; + } + // Reads stay on the retained generation; commit refreshes may follow data-only snapshots, + // while writer-contract changes still invalidate files produced for the retained metadata. + return IcebergSnapshotCacheValue.createWritableTable( + retainedTable, IcebergUtils.getIcebergTable(dorisTable)); + } + + /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ + public void beginMerge(ExternalTable dorisTable, Table targetTable) throws UserException { try { ops.getExecutionAuthenticator().execute(() -> { this.branchName = null; - this.table = IcebergUtils.getIcebergTable(dorisTable); + // Keep row binding, partition routing, writer schema, and commit on one generation. + this.table = targetTable; this.baseSnapshotId = getSnapshotIdIfPresent(table); if (table instanceof org.apache.iceberg.HasTableOperations) { int formatVersion = ((org.apache.iceberg.HasTableOperations) table).operations() @@ -306,7 +318,7 @@ public void beginMerge(ExternalTable dorisTable) throws UserException { + " must have format version 2 or higher for position deletes"); } } - this.transaction = table.newTransaction(); + this.transaction = createTransactionTable(dorisTable, table).newTransaction(); this.rewrittenDeleteFilesByReferencedDataFile = Collections.emptyMap(); LOG.info("Started merge transaction for table: {}", dorisTable.getName()); return null; @@ -913,7 +925,7 @@ private void commitStaticPartitionOverwrite(List pendingResults) { * @param schema Schema of the table * @return Iceberg Expression for partition filtering */ - private Expression buildPartitionFilter( + Expression buildPartitionFilter( Map staticPartitions, PartitionSpec spec, Schema schema) { @@ -922,6 +934,7 @@ private Expression buildPartitionFilter( } List predicates = new ArrayList<>(); + HashSet matchedPartitionNames = new HashSet<>(); for (PartitionField field : spec.fields()) { String partitionColName = field.name(); @@ -949,11 +962,18 @@ private Expression buildPartitionFilter( eqExpr = Expressions.equal(sourceColName, partitionValue); } predicates.add(eqExpr); + matchedPartitionNames.add(partitionColName); } } - if (predicates.isEmpty()) { - return Expressions.alwaysTrue(); + if (matchedPartitionNames.size() != staticPartitions.size()) { + HashSet unmatchedPartitionNames = new HashSet<>(staticPartitions.keySet()); + unmatchedPartitionNames.removeAll(matchedPartitionNames); + // Falling back to alwaysTrue here would turn a misspelled/static stale key into a + // whole-table overwrite, so every user-specified key must bind to the retained spec. + throw new IllegalArgumentException( + "Static partition columns are not present in the Iceberg partition spec: " + + unmatchedPartitionNames); } // Combine all predicates with AND diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 5022495d993994..cc46a88928ab20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -690,8 +690,11 @@ public static Type icebergTypeToDorisType(org.apache.iceberg.types.Type type, bo case STRUCT: Types.StructType struct = (Types.StructType) type; ArrayList nestedTypes = struct.fields().stream().map( + // Nested docs live on Iceberg fields, so carry them into the Doris type; + // otherwise DESC can only expose the top-level column comment. x -> new StructField(x.name(), - icebergTypeToDorisType(x.type(), enableMappingVarbinary, enableMappingTimestampTz))) + icebergTypeToDorisType(x.type(), enableMappingVarbinary, enableMappingTimestampTz), + x.doc(), x.isOptional())) .collect(Collectors.toCollection(ArrayList::new)); return new StructType(nestedTypes); case VARIANT: @@ -1448,12 +1451,6 @@ public static IcebergTableQueryInfo getQuerySpecSnapshot( refName = params.getListParams().get(0); } SnapshotRef snapshotRef = table.refs().get(refName); - LOG.info("[BranchDebug] getQuerySpecSnapshot: refName={}, snapshotId={}, " - + "currentSnapshotId={}, allRefs={}", - refName, - snapshotRef != null ? snapshotRef.snapshotId() : "null", - table.currentSnapshot() != null ? table.currentSnapshot().snapshotId() : "null", - table.refs()); if (params.isBranch()) { if (snapshotRef == null || !snapshotRef.isBranch()) { throw new UserException("Table " + table.name() + " does not have branch named " + refName); @@ -1466,6 +1463,7 @@ public static IcebergTableQueryInfo getQuerySpecSnapshot( return new IcebergTableQueryInfo( snapshotRef.snapshotId(), refName, + // Branches use the table's current schema while tags retain their snapshot schema. SnapshotUtil.schemaFor(table, refName).schemaId()); } @@ -1489,9 +1487,11 @@ public static IcebergTableQueryInfo getQuerySpecSnapshot( if (!table.refs().containsKey(value)) { throw new UserException("Table " + table.name() + " does not have tag or branch named " + value); } + SnapshotRef snapshotRef = table.refs().get(value); return new IcebergTableQueryInfo( - table.refs().get(value).snapshotId(), + snapshotRef.snapshotId(), value, + // VERSION must preserve Iceberg's branch-current/tag-historical schema contract. SnapshotUtil.schemaFor(table, value).schemaId() ); } else { @@ -1821,7 +1821,7 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( Optional scanParams) { if (tableSnapshot.isPresent() || IcebergUtils.isIcebergBranchOrTag(scanParams)) { // If a snapshot is specified, use the specified snapshot and the corresponding schema (not latest). - Table icebergTable = getIcebergTable(dorisTable); + Table icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(getIcebergTable(dorisTable)); IcebergTableQueryInfo info; try { info = getQuerySpecSnapshot(icebergTable, tableSnapshot, scanParams); @@ -1831,19 +1831,29 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), - getNameMapping(icebergTable)); + getNameMapping(icebergTable), + icebergTable); } return getLatestSnapshotCacheValue(dorisTable); } public static List getIcebergSchema(ExternalTable dorisTable) { - Optional snapshotFromContext = MvccUtil.getSnapshotFromContext(dorisTable); - IcebergSnapshotCacheValue cacheValue = IcebergUtils.getSnapshotCacheValue(snapshotFromContext, dorisTable); + return getIcebergSchema(dorisTable, MvccUtil.getSnapshotFromContext(dorisTable)); + } + + public static List getIcebergSchema(ExternalTable dorisTable, Optional snapshot) { + IcebergSnapshotCacheValue cacheValue = IcebergUtils.getSnapshotCacheValue(snapshot, dorisTable); return IcebergUtils.getSchemaCacheValue(dorisTable, cacheValue).getSchema(); } public static List getIcebergPartitionColumns(Optional snapshot, ExternalTable dorisTable) { IcebergSnapshotCacheValue snapshotValue = getSnapshotCacheValue(snapshot, dorisTable); + if (snapshotValue.getIcebergTable().isPresent()) { + // Schema ID alone cannot identify the partition spec; metadata-only evolution may keep + // the same schema and snapshot IDs while changing spec(), so derive both from T0. + return buildTableSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId(), + snapshotValue.getIcebergTable().get()).getPartitionColumns(); + } return getSchemaCacheValue(dorisTable, snapshotValue).getPartitionColumns(); } @@ -1870,6 +1880,11 @@ private static Optional loadViewSchemaCacheValue(ExternalTable private static Optional loadTableSchemaCacheValue(ExternalTable dorisTable, long schemaId) { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + return Optional.of(buildTableSchemaCacheValue(dorisTable, schemaId, icebergTable)); + } + + private static IcebergSchemaCacheValue buildTableSchemaCacheValue(ExternalTable dorisTable, long schemaId, + Table icebergTable) { List schema = IcebergUtils.getSchema(dorisTable, schemaId, false, icebergTable); // get table partition column info List tmpColumns = Lists.newArrayList(); @@ -1883,7 +1898,7 @@ private static Optional loadTableSchemaCacheValue(ExternalTabl } } } - return Optional.of(new IcebergSchemaCacheValue(schema, tmpColumns)); + return new IcebergSchemaCacheValue(schema, tmpColumns); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java index 74afd0052b4721..095bad988849ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutor.java @@ -20,7 +20,9 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.UserException; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.qe.ConnectContext; import org.apache.doris.resource.computegroup.ComputeGroup; import org.apache.doris.scheduler.exception.JobException; @@ -28,10 +30,13 @@ import org.apache.doris.system.Backend; import com.google.common.collect.Lists; +// Keep third-party imports lexical to preserve the repository's CustomImportOrder invariant. +import org.apache.iceberg.Table; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -60,7 +65,11 @@ public RewriteResult executeGroupsConcurrently(List groups, lo long transactionId = dorisTable.getCatalog().getTransactionManager().begin(); IcebergTransaction transaction = (IcebergTransaction) dorisTable.getCatalog().getTransactionManager() .getTransaction(transactionId); - transaction.beginRewrite(dorisTable); + MvccSnapshot targetSnapshot = dorisTable.loadSnapshot(Optional.empty(), Optional.empty()); + Table targetIcebergTable = ((IcebergMvccSnapshot) targetSnapshot).getSnapshotCacheValue() + .getIcebergTable().orElseThrow( + () -> new UserException("Iceberg rewrite target metadata is not available")); + transaction.beginRewrite(dorisTable, targetIcebergTable); // Register files to delete for (RewriteDataGroup group : groups) { @@ -82,6 +91,7 @@ public RewriteResult executeGroupsConcurrently(List groups, lo group, transactionId, dorisTable, + targetSnapshot, connectContext, targetFileSizeBytes, availableBeCount, @@ -121,13 +131,18 @@ public void onTaskFailed(Long taskId, Exception error) { long rewrittenBytesCount = groups.stream().mapToLong(group -> group.getTotalSize()).sum(); int removedDeleteFilesCount = groups.stream().mapToInt(group -> group.getDeleteFileCount()).sum(); - // Commit transaction - transaction.commit(); + commitAndInvalidate(transaction); return new RewriteResult(rewrittenDataFilesCount, addedDataFilesCount, rewrittenBytesCount, removedDeleteFilesCount); } + void commitAndInvalidate(IcebergTransaction transaction) throws UserException { + transaction.commit(); + // Rewrite commits bypass the external-table DDL path, so evict the pre-rewrite snapshot before reuse. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable); + } + /** * Wait for all tasks to complete using notification mechanism */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java index eee1a7eb60256b..d20931a83b309c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/rewrite/RewriteGroupTask.java @@ -21,6 +21,8 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Status; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccTableInfo; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundRelation; @@ -58,6 +60,7 @@ public class RewriteGroupTask implements TransientTaskExecutor { private final RewriteDataGroup group; private final long transactionId; private final IcebergExternalTable dorisTable; + private final MvccSnapshot targetSnapshot; private final ConnectContext connectContext; private final long targetFileSizeBytes; private final RewriteResultCallback resultCallback; @@ -72,6 +75,7 @@ public class RewriteGroupTask implements TransientTaskExecutor { public RewriteGroupTask(RewriteDataGroup group, long transactionId, IcebergExternalTable dorisTable, + MvccSnapshot targetSnapshot, ConnectContext connectContext, long targetFileSizeBytes, int availableBeCount, @@ -79,6 +83,7 @@ public RewriteGroupTask(RewriteDataGroup group, this.group = group; this.transactionId = transactionId; this.dorisTable = dorisTable; + this.targetSnapshot = targetSnapshot; this.connectContext = connectContext; this.targetFileSizeBytes = targetFileSizeBytes; this.availableBeCount = availableBeCount; @@ -88,6 +93,18 @@ public RewriteGroupTask(RewriteDataGroup group, this.isFinished = new AtomicBoolean(false); } + // Tests that only exercise scheduling strategy do not create an Iceberg metadata snapshot. + RewriteGroupTask(RewriteDataGroup group, + long transactionId, + IcebergExternalTable dorisTable, + ConnectContext connectContext, + long targetFileSizeBytes, + int availableBeCount, + RewriteResultCallback resultCallback) { + this(group, transactionId, dorisTable, null, connectContext, targetFileSizeBytes, + availableBeCount, resultCallback); + } + @Override public Long getId() { return taskId; @@ -259,6 +276,12 @@ private ConnectContext buildConnectContext() { statementContext.setConnectContext(taskContext); taskContext.setStatementContext(statementContext); + if (targetSnapshot != null) { + // Every group binds both its source relation and write sink to the transaction's + // retained Iceberg generation instead of independently observing a catalog refresh. + statementContext.setSnapshot(new MvccTableInfo(dorisTable), targetSnapshot); + } + // Set GATHER distribution flag if needed (for small data rewrite) statementContext.setUseGatherForIcebergRewrite(strategy.useGather); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 0eae2b93736550..fb532f506908b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -41,6 +41,8 @@ import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader; @@ -85,6 +87,8 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -97,6 +101,7 @@ import org.apache.iceberg.SplittableScanTask; import org.apache.iceberg.Table; import org.apache.iceberg.TableScan; +import org.apache.iceberg.expressions.Binder; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.InclusiveMetricsEvaluator; @@ -105,6 +110,7 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.ScanTaskUtil; import org.apache.iceberg.util.SerializationUtil; @@ -118,11 +124,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; @@ -239,7 +247,9 @@ private void initIcebergSource(ExternalTable table) { protected void doInitialize() throws UserException { long startTime = System.currentTimeMillis(); try { + getRelationSnapshot(); icebergTable = source.getIcebergTable(); + icebergTable = useFrozenTableGeneration(icebergTable); partitionMapInfos = new HashMap<>(); isPartitionedTable = icebergTable.spec().isPartitioned(); // Metadata tables (system tables) are not BaseTable instances, so we need to handle this case @@ -266,7 +276,7 @@ protected void doInitialize() throws UserException { } private Optional>> extractNameMapping() { - Optional snapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + Optional snapshot = getPinnedRelationSnapshot(); if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { // The mapping must come from the same metadata generation as the pinned schema; a // property-only refresh can otherwise change alias semantics within one statement. @@ -275,6 +285,13 @@ private Optional>> extractNameMapping() { return IcebergUtils.getNameMapping(icebergTable); } + private Optional getPinnedRelationSnapshot() { + Optional snapshot = getRelationSnapshot(); + // Legacy planner paths do not transport relation-local metadata, but their statement + // context still owns the same fence; use it only when the scan itself has none. + return snapshot.isPresent() ? snapshot : MvccUtil.getSnapshotFromContext(source.getTargetTable()); + } + @Override protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { if (split instanceof IcebergSplit) { @@ -499,7 +516,10 @@ void initializeIcebergSchemaInfo(Optional>> nameMappin // historical names, types, and initial defaults when an old data file lacks such a key. // An identity partition column can also be a physical field in files written by an older // partition spec, so preserving the complete schema is required for partition evolution. - ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), + List columns = source.getTargetTable() instanceof ExternalTable + ? ((ExternalTable) source.getTargetTable()).getFullSchema(getPinnedRelationSnapshot()) + : source.getTargetTable().getColumns(); + ExternalUtil.initSchemaInfoForAllColumn(params, -1L, columns, nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), getBase64EncodedInitialDefaultsForScan()); } @@ -520,10 +540,10 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); } IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); - Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); Schema scanSchema = null; - if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { - long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) + Optional snapshot = getPinnedRelationSnapshot(); + if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { + long schemaId = ((IcebergMvccSnapshot) snapshot.get()) .getSnapshotCacheValue().getSnapshot().getSchemaId(); scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); } else { @@ -638,6 +658,7 @@ public TableScan createTableScan() throws UserException { return icebergTableScan; } + icebergTable = useFrozenTableGeneration(icebergTable); TableScan scan = icebergTable.newScan().metricsReporter(new IcebergMetricsReporter()); // set snapshot @@ -648,12 +669,22 @@ public TableScan createTableScan() throws UserException { } else { scan = scan.useSnapshot(info.getSnapshotId()); } + if (!isSystemTable) { + Schema selectedSchema = icebergTable.schemas().get(info.getSchemaId()); + // The snapshot fences data, while this projection preserves the schema bound for a + // branch instead of letting useSnapshot replace it with the snapshot's old schema. + scan = scan.project(Preconditions.checkNotNull( + selectedSchema, "Schema %s for Iceberg scan is null", info.getSchemaId())); + } } + Schema scanSchema = scan.schema(); // set filter List expressions = new ArrayList<>(); for (Expr conjunct : conjuncts) { - Expression expression = IcebergUtils.convertToIcebergExpr(conjunct, icebergTable.schema()); + // Ref/snapshot selection can change field IDs and names, so every pruning expression + // must be bound against the schema selected by this scan rather than the current table. + Expression expression = IcebergUtils.convertToIcebergExpr(conjunct, scanSchema); if (expression != null) { expressions.add(expression); } @@ -668,6 +699,82 @@ public TableScan createTableScan() throws UserException { return icebergTableScan; } + private Table useFrozenTableGeneration(Table currentTable) { + Optional snapshot = getPinnedRelationSnapshot(); + if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { + IcebergSnapshotCacheValue cacheValue = + ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); + if (cacheValue.getIcebergTable().isPresent()) { + Table frozenBaseTable = cacheValue.getIcebergTable().get(); + if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { + IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); + if (systemTable.supportsSnapshotSelection()) { + MetadataTableType tableType = MetadataTableType.from(systemTable.getSysTableType()); + Preconditions.checkArgument(tableType != null, + "Unknown Iceberg system table type: %s", systemTable.getSysTableType()); + // Snapshot-selectable metadata tables must derive their scans and schemas + // from the same frozen base generation as the relation's snapshot fence. + return MetadataTableUtils.createMetadataTableInstance(frozenBaseTable, tableType); + } + return currentTable; + } + // Snapshot selection fences data files, but spec, properties, expiration state, + // and schema lookup still come from Table; keep them on the bound generation too. + return frozenBaseTable; + } + } + return currentTable; + } + + @VisibleForTesting + Schema getSystemTableProjectedSchema(List expressions, boolean caseSensitive) + throws UserException { + List projectedFields = new ArrayList<>(); + Set projectedFieldIds = new HashSet<>(); + List partitionKeys = getPathPartitionKeys(); + for (SlotDescriptor slot : desc.getSlots()) { + Column column = slot.getColumn(); + String columnName = column.getName(); + if (!isFileSlot(classifyColumn(slot, partitionKeys))) { + continue; + } + + NestedField field = caseSensitive + ? icebergTable.schema().findField(columnName) + : icebergTable.schema().caseInsensitiveFindField(columnName); + if (field == null) { + throw new UserException("Column " + columnName + " not found in Iceberg system table schema"); + } + if (projectedFieldIds.add(field.fieldId())) { + projectedFields.add(field); + } + } + + Set filterFieldIds = Binder.boundReferences( + icebergTable.schema().asStruct(), expressions, caseSensitive); + for (Integer fieldId : filterFieldIds) { + NestedField field = getTopLevelSystemTableField(fieldId); + if (field == null) { + throw new UserException( + "Column with field id " + fieldId + " not found in Iceberg system table schema"); + } + if (!projectedFieldIds.contains(field.fieldId())) { + throw new UserException("Iceberg system table filter column " + field.name() + + " is not materialized by the planner"); + } + } + return new Schema(projectedFields); + } + + private NestedField getTopLevelSystemTableField(int fieldId) { + for (NestedField field : icebergTable.schema().columns()) { + if (field.fieldId() == fieldId || TypeUtil.getProjectedIds(field.type()).contains(fieldId)) { + return field; + } + } + return null; + } + private CloseableIterable planFileScanTask(TableScan scan) { if (!IcebergUtils.isManifestCacheEnabled(source.getCatalog())) { return splitFiles(scan); @@ -746,11 +853,12 @@ private CloseableIterable planFileScanTaskWithManifestCache(TableS throw new RuntimeException("Iceberg scan target table is not an external table"); } ExternalTable targetExternalTable = (ExternalTable) source.getTargetTable(); + Schema scanSchema = scan.schema(); // Convert query conjuncts to Iceberg filter expression // This combines all predicates with AND logic for partition/file pruning Expression filterExpr = conjuncts.stream() - .map(conjunct -> IcebergUtils.convertToIcebergExpr(conjunct, icebergTable.schema())) + .map(conjunct -> IcebergUtils.convertToIcebergExpr(conjunct, scanSchema)) .filter(Objects::nonNull) .reduce(Expressions.alwaysTrue(), Expressions::and); @@ -766,7 +874,7 @@ private CloseableIterable planFileScanTaskWithManifestCache(TableS // Create metrics evaluator for file-level pruning based on column statistics InclusiveMetricsEvaluator metricsEvaluator = - new InclusiveMetricsEvaluator(icebergTable.schema(), filterExpr, caseSensitive); + new InclusiveMetricsEvaluator(scanSchema, filterExpr, caseSensitive); // ========== Phase 1: Load delete files from delete manifests ========== List deleteFiles = new ArrayList<>(); @@ -843,7 +951,7 @@ private CloseableIterable planFileScanTaskWithManifestCache(TableS tasks.add(new BaseFileScanTask( dataFile, deletes.toArray(new DeleteFile[0]), - SchemaParser.toJson(icebergTable.schema()), + SchemaParser.toJson(scanSchema), PartitionSpecParser.toJson(spec), residualEvaluator)); } @@ -1215,10 +1323,11 @@ private List doGetPositionDeletesSystemTableSplits() throws UserException scan = scan.useSnapshot(info.getSnapshotId()); } } + Schema scanSchema = scan.schema(); List expressions = new ArrayList<>(); for (Expr conjunct : conjuncts) { - Expression expression = IcebergUtils.convertToIcebergExpr(conjunct, icebergTable.schema()); + Expression expression = IcebergUtils.convertToIcebergExpr(conjunct, scanSchema); if (expression != null) { expressions.add(expression); } @@ -1335,6 +1444,24 @@ public IcebergTableQueryInfo getSpecifiedSnapshot() throws UserException { TableSnapshot tableSnapshot = getQueryTableSnapshot(); TableScanParams scanParams = getScanParams(); Optional params = Optional.ofNullable(scanParams); + Optional snapshot = getPinnedRelationSnapshot(); + if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { + if (source.getTargetTable() instanceof IcebergSysExternalTable + && !((IcebergSysExternalTable) source.getTargetTable()).supportsSnapshotSelection()) { + return null; + } + IcebergSnapshot pinnedSnapshot = + ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue().getSnapshot(); + // Empty Iceberg tables use -1 as a cache sentinel; passing it to useSnapshot would + // turn a valid empty scan (including metadata tables) into an unknown-snapshot error. + if (pinnedSnapshot.getSnapshotId() < 0) { + return null; + } + // Every bound relation, including latest and metadata tables, must keep the data fence + // captured during analysis instead of consulting a newer current snapshot at execution. + return new IcebergTableQueryInfo( + pinnedSnapshot.getSnapshotId(), null, Math.toIntExact(pinnedSnapshot.getSchemaId())); + } if (tableSnapshot != null || IcebergUtils.isIcebergBranchOrTag(params)) { return IcebergUtils.getQuerySpecSnapshot( icebergTable, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/EmptyMvccSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/EmptyMvccSnapshot.java index 35f63291a258c8..43f8ecf73c7c95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/EmptyMvccSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/EmptyMvccSnapshot.java @@ -18,4 +18,8 @@ package org.apache.doris.datasource.mvcc; public class EmptyMvccSnapshot implements MvccSnapshot { + @Override + public boolean isSameSnapshot(MvccSnapshot other) { + return other instanceof EmptyMvccSnapshot; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java index d7826b0a5de19e..221932da8c2e07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java @@ -22,4 +22,12 @@ * but it should be ensured that the table information queried through this snapshot remains unchanged */ public interface MvccSnapshot { + /** + * Whether two independently resolved wrappers identify the same immutable metadata state. + */ + default boolean isSameSnapshot(MvccSnapshot other) { + // Unknown implementations remain identity-conservative so optimizer rewrites cannot + // merge scans unless the provider exposes a stable snapshot identity. + return this == other; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index df384d4d4d0071..403c3bb0fb6ea0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -57,6 +57,7 @@ import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.DataTable; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.Split; import org.apache.paimon.types.DataField; @@ -200,12 +201,13 @@ private PaimonSnapshotCacheValue getPaimonSnapshotCacheValue(Optional getFullSchema() { - return getPaimonSchemaCacheValue(MvccUtil.getSnapshotFromContext(this)).getSchema(); + return getFullSchema(MvccUtil.getSnapshotFromContext(this)); + } + + @Override + public List getFullSchema(Optional snapshot) { + return getPaimonSchemaCacheValue(snapshot).getSchema(); } @Override @@ -377,29 +384,7 @@ public Optional initSchema(SchemaCacheKey key) { makeSureInitialized(); PaimonSchemaCacheKey paimonSchemaCacheKey = (PaimonSchemaCacheKey) key; try { - Table table = getBasePaimonTable(); - TableSchema tableSchema = ((DataTable) table).schemaManager().schema(paimonSchemaCacheKey.getSchemaId()); - List columns = tableSchema.fields(); - List dorisColumns = Lists.newArrayListWithCapacity(columns.size()); - Set partitionColumnNames = Sets.newHashSet(tableSchema.partitionKeys()); - List partitionColumns = Lists.newArrayList(); - for (DataField field : columns) { - Column column = new Column(field.name(), - PaimonUtil.paimonTypeToDorisType(field.type(), getCatalog().getEnableMappingVarbinary(), - getCatalog().getEnableMappingTimestampTz()), - true, - null, true, field.description(), true, - -1); - PaimonUtil.updatePaimonColumnUniqueId(column, field); - if (field.type().getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { - column.setWithTZExtraInfo(); - } - dorisColumns.add(column); - if (partitionColumnNames.contains(field.name())) { - partitionColumns.add(column); - } - } - return Optional.of(new PaimonSchemaCacheValue(dorisColumns, partitionColumns, tableSchema)); + return Optional.of(loadSchema((DataTable) getBasePaimonTable(), paimonSchemaCacheKey.getSchemaId())); } catch (Exception e) { throw new CacheException("failed to initSchema for: %s.%s.%s.%s", null, getCatalog().getName(), key.getNameMapping().getLocalDbName(), @@ -415,9 +400,43 @@ public Optional getSchemaCacheValue() { private PaimonSchemaCacheValue getPaimonSchemaCacheValue(Optional snapshot) { PaimonSnapshotCacheValue snapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot); + if (snapshotCacheValue.isSchemaFromSnapshotTable()) { + PaimonSnapshot paimonSnapshot = snapshotCacheValue.getSnapshot(); + // The snapshot table already carries the branch-specific schema; looking it up by id + // can accidentally use the base table's schema namespace. + return loadSchema(((FileStoreTable) paimonSnapshot.getTable()).schema()); + } return PaimonUtils.getSchemaCacheValue(this, snapshotCacheValue); } + private PaimonSchemaCacheValue loadSchema(DataTable table, long schemaId) { + return loadSchema(table.schemaManager().schema(schemaId)); + } + + private PaimonSchemaCacheValue loadSchema(TableSchema tableSchema) { + List columns = tableSchema.fields(); + List dorisColumns = Lists.newArrayListWithCapacity(columns.size()); + Set partitionColumnNames = Sets.newHashSet(tableSchema.partitionKeys()); + List partitionColumns = Lists.newArrayList(); + for (DataField field : columns) { + Column column = new Column(field.name(), + PaimonUtil.paimonTypeToDorisType(field.type(), getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz()), + true, + null, true, field.description(), true, + -1); + PaimonUtil.updatePaimonColumnUniqueId(column, field); + if (field.type().getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + column.setWithTZExtraInfo(); + } + dorisColumns.add(column); + if (partitionColumnNames.contains(field.name())) { + partitionColumns.add(column); + } + } + return new PaimonSchemaCacheValue(dorisColumns, partitionColumns, tableSchema); + } + private PaimonSnapshotCacheValue getOrFetchSnapshotCacheValue(Optional snapshot) { if (snapshot.isPresent()) { return ((PaimonMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java index 2307e91adb3911..90f87bde4d0ae3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMvccSnapshot.java @@ -29,4 +29,14 @@ public PaimonMvccSnapshot(PaimonSnapshotCacheValue snapshotCacheValue) { public PaimonSnapshotCacheValue getSnapshotCacheValue() { return snapshotCacheValue; } + + @Override + public boolean isSameSnapshot(MvccSnapshot other) { + if (!(other instanceof PaimonMvccSnapshot)) { + return false; + } + PaimonSnapshot left = snapshotCacheValue.getSnapshot(); + PaimonSnapshot right = ((PaimonMvccSnapshot) other).snapshotCacheValue.getSnapshot(); + return left.getSnapshotId() == right.getSnapshotId() && left.getSchemaId() == right.getSchemaId(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index c50ecdabfde3df..37be7c6a5f3585 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -21,10 +21,17 @@ public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; private final PaimonSnapshot snapshot; + private final boolean schemaFromSnapshotTable; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { + this(partitionInfo, snapshot, false); + } + + public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, + boolean schemaFromSnapshotTable) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; + this.schemaFromSnapshotTable = schemaFromSnapshotTable; } public PaimonPartitionInfo getPartitionInfo() { @@ -34,4 +41,8 @@ public PaimonPartitionInfo getPartitionInfo() { public PaimonSnapshot getSnapshot() { return snapshot; } + + public boolean isSchemaFromSnapshotTable() { + return schemaFromSnapshotTable; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index c7b708e9742a36..84131773743028 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -31,8 +31,12 @@ import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.credentials.CredentialUtils; import org.apache.doris.datasource.credentials.VendedCredentialsFactory; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonMvccSnapshot; import org.apache.doris.datasource.paimon.PaimonScanParams; +import org.apache.doris.datasource.paimon.PaimonSnapshot; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; import org.apache.doris.datasource.paimon.PaimonUtil; import org.apache.doris.datasource.paimon.PaimonUtils; @@ -62,6 +66,7 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; +import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.DataSplit; @@ -184,7 +189,11 @@ public PaimonScanNode(PlanNodeId id, @Override protected void doInitialize() throws UserException { - if (source == null) { + Optional relationSnapshot = getRelationSnapshot(); + if (desc.getTable() instanceof PaimonExternalTable) { + // Rebuild the source before applying query options so both layers use this relation's snapshot. + source = new PaimonSource(desc, relationSnapshot); + } else if (source == null) { source = new PaimonSource(desc); } processedTable = getProcessedTable(); @@ -290,7 +299,14 @@ private void putHistorySchemaInfo(Long schemaId) { } } - TableSchema tableSchema = PaimonUtils.getSchemaCacheValue(targetTable, schemaId).getTableSchema(); + TableSchema tableSchema; + if (targetTable instanceof PaimonExternalTable) { + // Schema IDs are scoped to the resolved relation table, so a branch ID must + // never be looked up through the base table's schema cache namespace. + tableSchema = ((DataTable) source.getPaimonTable()).schemaManager().schema(schemaId); + } else { + tableSchema = PaimonUtils.getSchemaCacheValue(targetTable, schemaId).getTableSchema(); + } params.addToHistorySchemaInfo(PaimonUtil.getHistorySchemaInfo(targetTable, tableSchema, source.getCatalog().getEnableMappingVarbinary(), source.getCatalog().getEnableMappingTimestampTz())); @@ -656,6 +672,14 @@ public Map getIncrReadParams() throws UserException { public List getPaimonSplitFromAPI() throws UserException { long startTime = System.currentTimeMillis(); try { + Optional relationSnapshot = getRelationSnapshot(); + if (relationSnapshot.isPresent() && relationSnapshot.get() instanceof PaimonMvccSnapshot + && ((PaimonMvccSnapshot) relationSnapshot.get()).getSnapshotCacheValue() + .getSnapshot().getSnapshotId() == PaimonSnapshot.INVALID_SNAPSHOT_ID) { + // An empty snapshot is a bound generation: consulting the live table here could + // expose the first commit made after analysis completed. + return Collections.emptyList(); + } Table paimonTable = getProcessedTable(); Map resolvedOptions = scanParams == null ? Collections.emptyMap() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java index e8a3fb33d957ed..316379a79f1dfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java @@ -26,6 +26,7 @@ import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonScanParams; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; import org.apache.doris.thrift.TFileAttributes; @@ -33,6 +34,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; +import java.util.Map; import java.util.Optional; public class PaimonSource { @@ -48,9 +50,13 @@ public PaimonSource() { } public PaimonSource(TupleDescriptor desc) { + this(desc, MvccUtil.getSnapshotFromContext((ExternalTable) desc.getTable())); + } + + public PaimonSource(TupleDescriptor desc, Optional snapshot) { this.desc = desc; this.paimonExtTable = (ExternalTable) desc.getTable(); - this.originTable = resolvePaimonTable(paimonExtTable); + this.originTable = resolvePaimonTable(paimonExtTable, snapshot); } public TupleDescriptor getDesc() { @@ -63,6 +69,15 @@ public Table getPaimonTable() { public Table getPaimonTable(TableScanParams scanParams) { if (paimonExtTable instanceof PaimonExternalTable) { + if (scanParams != null && scanParams.isOptions() + && PaimonScanParams.usesStatementSnapshot(scanParams.getMapParams()) + && !PaimonScanParams.selectsSchema(scanParams.getMapParams())) { + Map resolvedOptions = scanParams.getOrResolveMapParams( + options -> PaimonScanParams.resolveOptions(originTable, options)); + // Behavioral OPTIONS must decorate this relation's retained table; consulting the + // statement cache here can borrow another relation's historical generation. + return PaimonScanParams.applyOptions(originTable, resolvedOptions); + } return ((PaimonExternalTable) paimonExtTable).getPaimonTable(scanParams); } if (paimonExtTable instanceof PaimonSysExternalTable) { @@ -80,8 +95,7 @@ public ExternalTable getExternalTable() { return paimonExtTable; } - private Table resolvePaimonTable(ExternalTable table) { - Optional snapshot = MvccUtil.getSnapshotFromContext(table); + private Table resolvePaimonTable(ExternalTable table, Optional snapshot) { if (table instanceof PaimonExternalTable) { return ((PaimonExternalTable) table).getPaimonTable(snapshot); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index c4b1292688255e..6d2634645f3a34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -266,6 +266,7 @@ public enum TableFrom { private Backend groupCommitMergeBackend; private final Map snapshots = Maps.newHashMap(); + private final Map latestSnapshots = Maps.newHashMap(); // Record external tables that can be preloaded before internal table locks are acquired. private final Map externalTablePreloadInfos = new LinkedHashMap<>(); private ExternalMetadataPreloadResult externalMetadataPreloadResult; @@ -937,15 +938,23 @@ public void addPlannerHook(PlannerHook plannerHook) { * @param tableSnapshot table snapshot info * @param scanParams table scan params (e.g., branch/tag for Iceberg tables) */ - public void loadSnapshots(TableIf specificTable, Optional tableSnapshot, + public Optional loadSnapshots(TableIf specificTable, Optional tableSnapshot, Optional scanParams) { - if (specificTable instanceof MvccTable) { - MvccTableInfo mvccTableInfo = new MvccTableInfo(specificTable); - if (!snapshots.containsKey(mvccTableInfo)) { - snapshots.put(mvccTableInfo, - ((MvccTable) specificTable).loadSnapshot(tableSnapshot, scanParams)); - } + if (!(specificTable instanceof MvccTable)) { + return Optional.empty(); } + MvccTableInfo mvccTableInfo = new MvccTableInfo(specificTable); + MvccSnapshot snapshot; + if (tableSnapshot.isPresent() || scanParams.isPresent()) { + snapshot = ((MvccTable) specificTable).loadSnapshot(tableSnapshot, scanParams); + } else { + // Keep latest metadata separate: a historical relation may temporarily become the + // table-scoped snapshot, but it must not redefine what a later latest relation sees. + snapshot = latestSnapshots.computeIfAbsent(mvccTableInfo, + key -> ((MvccTable) specificTable).loadSnapshot(tableSnapshot, scanParams)); + } + snapshots.put(mvccTableInfo, snapshot); + return Optional.of(snapshot); } /** @@ -969,6 +978,9 @@ public Optional getSnapshot(TableIf tableIf) { * @param snapshot snapshot */ public void setSnapshot(MvccTableInfo mvccTableInfo, MvccSnapshot snapshot) { + // Injected snapshots are execution fences (for example MTMV refresh); an unqualified + // relation must reuse that fence instead of treating the latest-snapshot cache as empty. + latestSnapshots.put(mvccTableInfo, snapshot); snapshots.put(mvccTableInfo, snapshot); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java index 213baccafb2688..7c75c7fd166b2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java @@ -38,6 +38,7 @@ */ public class UnboundIcebergTableSink extends UnboundBaseExternalTableSink { private boolean rewrite = false; + private final Optional branchName; // Static partition key-value pairs for INSERT OVERWRITE ... PARTITION // (col='val', ...) @@ -97,12 +98,31 @@ public UnboundIcebergTableSink(List nameParts, CHILD_TYPE child, Map staticPartitionKeyValues, boolean rewrite) { + this(nameParts, colNames, hints, partitions, dmlCommandType, groupExpression, + logicalProperties, child, staticPartitionKeyValues, rewrite, Optional.empty()); + } + + /** + * constructor with static partition and branch + */ + public UnboundIcebergTableSink(List nameParts, + List colNames, + List hints, + List partitions, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child, + Map staticPartitionKeyValues, + boolean rewrite, + Optional branchName) { super(nameParts, PlanType.LOGICAL_UNBOUND_ICEBERG_TABLE_SINK, ImmutableList.of(), groupExpression, logicalProperties, colNames, dmlCommandType, child, hints, partitions); this.staticPartitionKeyValues = staticPartitionKeyValues != null ? ImmutableMap.copyOf(staticPartitionKeyValues) : null; this.rewrite = rewrite; + this.branchName = branchName; } public Map getStaticPartitionKeyValues() { @@ -118,7 +138,8 @@ public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "UnboundIcebergTableSink only accepts one child"); return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, - dmlCommandType, groupExpression, Optional.empty(), children.get(0), staticPartitionKeyValues, rewrite); + dmlCommandType, groupExpression, Optional.empty(), children.get(0), + staticPartitionKeyValues, rewrite, branchName); } @Override @@ -130,17 +151,28 @@ public R accept(PlanVisitor visitor, C context) { public Plan withGroupExpression(Optional groupExpression) { return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child(), - staticPartitionKeyValues, rewrite); + staticPartitionKeyValues, rewrite, branchName); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, - dmlCommandType, groupExpression, logicalProperties, children.get(0), staticPartitionKeyValues, rewrite); + dmlCommandType, groupExpression, logicalProperties, children.get(0), + staticPartitionKeyValues, rewrite, branchName); } public boolean isRewrite() { return rewrite; } + + public Optional getBranchName() { + return branchName; + } + + public UnboundIcebergTableSink withBranchName(Optional branchName) { + return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, Optional.empty(), child(), + staticPartitionKeyValues, rewrite, branchName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 44ff35ab1d52a3..a704909a9f35dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -613,7 +613,9 @@ public PlanFragment visitPhysicalIcebergTableSink(PhysicalIcebergTableSink outputExprs = Lists.newArrayList(); icebergTableSink.getOutput().stream().map(Slot::getExprId) .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId))); - IcebergTableSink sink = new IcebergTableSink((IcebergExternalTable) icebergTableSink.getTargetTable()); + IcebergTableSink sink = new IcebergTableSink( + (IcebergExternalTable) icebergTableSink.getTargetTable(), + icebergTableSink.getTargetIcebergTable()); rootFragment.setSink(sink); sink.setOutputExprs(outputExprs); return rootFragment; @@ -655,6 +657,7 @@ public PlanFragment visitPhysicalIcebergDeleteSink(PhysicalIcebergDeleteSink) sink).withBranchName(branchName); + } Optional cte = Optional.empty(); if (ctx.cte() != null) { cte = Optional.ofNullable(withCte(plan, ctx.cte())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index cd4eb0d328bccf..40e52b3921f171 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -43,6 +43,8 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonScanParams; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; @@ -408,7 +410,7 @@ public static LogicalPlan checkAndAddDeleteSignFilter(LogicalOlapScan scan, Conn } private Optional handleMetaTable(TableIf table, UnboundRelation unboundRelation, - List qualifiedTableName) { + List qualifiedTableName, CascadesContext cascadesContext) { Optional sysTablePlanOpt = SysTableResolver.resolveForPlan( table, qualifiedTableName.get(0), qualifiedTableName.get(1), qualifiedTableName.get(2)); if (!sysTablePlanOpt.isPresent()) { @@ -425,6 +427,14 @@ private Optional handleMetaTable(TableIf table, UnboundRelation unb validatePaimonSystemTableScanParams( (PaimonSysExternalTable) sysExternalTable, unboundRelation.getScanParams()); } + TableIf snapshotTable = sysExternalTable instanceof IcebergSysExternalTable + ? ((IcebergSysExternalTable) sysExternalTable).getSourceTable() + : sysExternalTable; + // A metadata-table scan reads base-table snapshots, so bind its fence from the base + // relation even though the synthetic system-table wrapper is not itself MVCC-aware. + Optional relationSnapshot = cascadesContext.getStatementContext().loadSnapshots( + snapshotTable, unboundRelation.getTableSnapshot(), + Optional.ofNullable(unboundRelation.getScanParams())); return Optional.of(new LogicalFileScan( unboundRelation.getRelationId(), sysExternalTable, @@ -433,7 +443,7 @@ private Optional handleMetaTable(TableIf table, UnboundRelation unb unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(), Optional.ofNullable(unboundRelation.getScanParams()), - Optional.empty())); + Optional.empty(), relationSnapshot)); } // TVF path: create table-valued function and return LogicalTVFRelation @@ -454,13 +464,14 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio // Handle meta table like "table_name$partitions" // qualifiedTableName should be like "ctl.db.tbl$partitions" - Optional logicalPlan = handleMetaTable(table, unboundRelation, qualifiedTableName); + Optional logicalPlan = handleMetaTable( + table, unboundRelation, qualifiedTableName, cascadesContext); if (logicalPlan.isPresent()) { return logicalPlan.get(); } List qualifierWithoutTableName = qualifiedTableName.subList(0, qualifiedTableName.size() - 1); - cascadesContext.getStatementContext().loadSnapshots( + Optional relationSnapshot = cascadesContext.getStatementContext().loadSnapshots( table, unboundRelation.getTableSnapshot(), Optional.ofNullable(unboundRelation.getScanParams())); @@ -491,7 +502,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio LogicalHudiScan hudiScan = new LogicalHudiScan(unboundRelation.getRelationId(), hmsTable, qualifierWithoutTableName, ImmutableList.of(), Optional.empty(), unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(), - Optional.empty()); + Optional.empty(), relationSnapshot); hudiScan = hudiScan.withScanParams( hmsTable, Optional.ofNullable(unboundRelation.getScanParams())); return hudiScan; @@ -501,7 +512,8 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio ImmutableList.of(), unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(), - Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty()); + Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty(), + relationSnapshot); } case ICEBERG_EXTERNAL_TABLE: IcebergExternalTable icebergExternalTable = (IcebergExternalTable) table; @@ -531,7 +543,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio qualifierWithoutTableName, ImmutableList.of(), unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(), - Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty()); + Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty(), relationSnapshot); case PAIMON_EXTERNAL_TABLE: case MAX_COMPUTE_EXTERNAL_TABLE: case TRINO_CONNECTOR_EXTERNAL_TABLE: @@ -540,7 +552,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio qualifierWithoutTableName, ImmutableList.of(), unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(), - Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty()); + Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty(), relationSnapshot); case DORIS_EXTERNAL_TABLE: ConnectContext ctx = cascadesContext.getConnectContext(); RemoteDorisExternalTable externalTable = (RemoteDorisExternalTable) table; @@ -558,7 +570,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio qualifierWithoutTableName, ImmutableList.of(), unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(), - Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty()); + Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty(), relationSnapshot); case SCHEMA: LogicalSchemaScan schemaScan = new LogicalSchemaScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 9e9d6079d29cc3..36464435448539 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -36,6 +36,8 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; @@ -44,6 +46,7 @@ import org.apache.doris.datasource.paimon.PaimonExternalDatabase; import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonWriteTarget; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -384,6 +387,14 @@ private static Map getColumnToOutput( MatchingContext> ctx, TableIf table, boolean isPartialUpdate, boolean isDeletePartialUpdate, LogicalTableSink boundSink, LogicalPlan child) { + return getColumnToOutput(ctx, table, isPartialUpdate, isDeletePartialUpdate, + boundSink, child, boundSink.getTargetTable().getFullSchema()); + } + + private static Map getColumnToOutput( + MatchingContext> ctx, + TableIf table, boolean isPartialUpdate, boolean isDeletePartialUpdate, + LogicalTableSink boundSink, LogicalPlan child, List targetSchema) { // we need to insert all the columns of the target table // although some columns are not mentions. // so we add a projects to supply the default value. @@ -399,7 +410,7 @@ private static Map getColumnToOutput( List materializedViewColumn = Lists.newArrayList(); List shadowColumns = Lists.newArrayList(); // generate slots not mentioned in sql, mv slots and shaded slots. - for (Column column : boundSink.getTargetTable().getFullSchema()) { + for (Column column : targetSchema) { if (column.isGeneratedColumn()) { generatedColumns.add(column); continue; @@ -726,6 +737,16 @@ private Plan bindIcebergTableSink(MatchingContext> IcebergExternalDatabase database = pair.first; IcebergExternalTable table = pair.second; LogicalPlan child = ((LogicalPlan) sink.child()); + Optional targetSnapshot = ctx.cascadesContext.getStatementContext() + .loadSnapshots(table, Optional.empty(), Optional.empty()); + IcebergSnapshotCacheValue targetSnapshotValue = ((IcebergMvccSnapshot) targetSnapshot.orElseThrow( + () -> new AnalysisException("Iceberg write target snapshot is not available"))) + .getSnapshotCacheValue(); + Table targetIcebergTable = targetSnapshotValue.getIcebergTable().orElseThrow( + () -> new AnalysisException("Iceberg write target metadata is not available")); + // Sink columns belong to the write target's latest snapshot. A historical child scan + // of the same table must not make target-column lookup use its older relation schema. + List targetSchema = table.getFullSchema(targetSnapshot); // Get static partition columns if present Map staticPartitions = sink.getStaticPartitionKeyValues(); @@ -735,7 +756,7 @@ private Plan bindIcebergTableSink(MatchingContext> // Validate static partition if present if (sink.hasStaticPartition()) { - validateStaticPartition(sink, table); + validateStaticPartition(sink, table, targetIcebergTable); } // Build bindColumns: exclude static partition columns from the columns that @@ -746,19 +767,19 @@ private Plan bindIcebergTableSink(MatchingContext> if (sink.getColNames().isEmpty()) { // When no column names specified, include all non-static-partition columns if (sink.isRewrite()) { - bindColumns = table.getBaseSchema(true).stream() + bindColumns = targetSchema.stream() .filter(col -> !staticPartitionColNames.contains(col.getName())) .filter(col -> col.isVisible() || IcebergUtils.isIcebergRowLineageColumn(col)) .collect(ImmutableList.toImmutableList()); } else { - bindColumns = table.getBaseSchema(true).stream() + bindColumns = targetSchema.stream() .filter(col -> !staticPartitionColNames.contains(col.getName())) .filter(Column::isVisible) .collect(ImmutableList.toImmutableList()); } } else { bindColumns = sink.getColNames().stream().map(cn -> { - Column column = table.getColumn(cn); + Column column = findColumn(targetSchema, cn); if (column == null) { throw new AnalysisException(String.format("column %s is not found in table %s", cn, table.getName())); @@ -774,6 +795,7 @@ private Plan bindIcebergTableSink(MatchingContext> LogicalIcebergTableSink boundSink = new LogicalIcebergTableSink<>( database, table, + targetIcebergTable, bindColumns, child.getOutput().stream() .map(NamedExpression.class::cast) @@ -791,7 +813,7 @@ private Plan bindIcebergTableSink(MatchingContext> } Map columnToOutput = getColumnToOutput(ctx, table, false, false, - boundSink, child); + boundSink, child, targetSchema); // For static partition columns, add constant expressions from PARTITION clause // This ensures partition column values are written to the data file @@ -799,7 +821,7 @@ private Plan bindIcebergTableSink(MatchingContext> for (Map.Entry entry : staticPartitions.entrySet()) { String colName = entry.getKey(); Expression valueExpr = entry.getValue(); - Column column = table.getColumn(colName); + Column column = findColumn(targetSchema, colName); if (column != null) { // Cast the literal to the correct column type Expression castExpr = TypeCoercionUtils.castIfNotSameType( @@ -809,7 +831,9 @@ private Plan bindIcebergTableSink(MatchingContext> } } - List insertSchema = table.getFullSchema(); + // Iceberg branches share the current table schema, while their rows stay pinned to the + // branch head; target coercion must therefore use latest metadata as well. + List insertSchema = targetSchema; if (!sink.isRewrite()) { insertSchema = insertSchema.stream() .filter(Column::isVisible) @@ -819,16 +843,23 @@ private Plan bindIcebergTableSink(MatchingContext> return boundSink.withChildAndUpdateOutput(fullOutputProject); } + private static Column findColumn(List schema, String columnName) { + return schema.stream() + .filter(column -> column.getName().equalsIgnoreCase(columnName)) + .findFirst() + .orElse(null); + } + /** * Validate static partition specification for Iceberg table */ - private void validateStaticPartition(UnboundIcebergTableSink sink, IcebergExternalTable table) { + private void validateStaticPartition(UnboundIcebergTableSink sink, IcebergExternalTable table, + Table icebergTable) { Map staticPartitions = sink.getStaticPartitionKeyValues(); if (staticPartitions == null || staticPartitions.isEmpty()) { return; } - Table icebergTable = table.getIcebergTable(); PartitionSpec partitionSpec = icebergTable.spec(); // Check if table is partitioned diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index d5cb735569d83a..3b975b60055369 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -414,7 +414,16 @@ && check(((MapType) originalType).getValueType(), ((MapType) targetType).getValu return false; } for (int i = 0; i < targetFields.size(); i++) { - if (originalFields.get(i).isNullable() != targetFields.get(i).isNullable()) { + // A nullable target can safely accept a required source, but the inverse would + // allow a possible NULL into a required nested field. + if (originalFields.get(i).isNullable() && !targetFields.get(i).isNullable()) { + return false; + } + // A required source field is not sufficient: value conversion itself can still + // produce NULL, so required targets only accept casts that preserve non-nullness. + if (!targetFields.get(i).isNullable() + && Cast.castNullable(false, originalFields.get(i).getDataType(), + targetFields.get(i).getDataType())) { return false; } if (!check(originalFields.get(i).getDataType(), targetFields.get(i).getDataType(), isStrictMode)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java index 2b258052dfd868..c53e07d86c973e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -43,7 +43,8 @@ public Rule build() { fileScan.getTableSample(), fileScan.getTableSnapshot(), fileScan.getOperativeSlots(), - fileScan.getScanParams()) + fileScan.getScanParams(), + fileScan.getRelationSnapshot()) ).toRule(RuleType.LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java index 97ed60c6078167..19a7af02c506ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java @@ -43,7 +43,8 @@ public Rule build() { fileScan.getTableSnapshot(), fileScan.getScanParams(), fileScan.getIncrementalRelation(), - fileScan.getOperativeSlots()) + fileScan.getOperativeSlots(), + fileScan.getRelationSnapshot()) ).toRule(RuleType.LOGICAL_HUDI_SCAN_TO_PHYSICAL_HUDI_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink.java index 73a7284b155eac..15c624deab7ab9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink.java @@ -36,6 +36,7 @@ public Rule build() { return new PhysicalIcebergDeleteSink<>( sink.getDatabase(), sink.getTargetTable(), + sink.getTargetIcebergTable(), sink.getCols(), sink.getOutputExprs(), sink.getDeleteContext(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java index 85b813b7fd60d7..072417d2ac6271 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java @@ -36,6 +36,7 @@ public Rule build() { return new PhysicalIcebergMergeSink<>( sink.getDatabase(), sink.getTargetTable(), + sink.getTargetIcebergTable(), sink.getCols(), sink.getOutputExprs(), sink.getDeleteContext(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java index c520ef83f2730c..6b5ebfa1c85d95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java @@ -36,6 +36,7 @@ public Rule build() { return new PhysicalIcebergTableSink<>( sink.getDatabase(), sink.getTargetTable(), + sink.getTargetIcebergTable(), sink.getCols(), sink.getOutputExprs(), Optional.empty(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java index 35a1105c916787..303a2187ca7ff9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.common.Pair; import org.apache.doris.datasource.ExternalTable; @@ -76,8 +77,8 @@ public Rule build() { private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable, LogicalFilter filter, LogicalFileScan scan, CascadesContext ctx) { Map selectedPartitionItems = Maps.newHashMap(); - if (CollectionUtils.isEmpty(externalTable.getPartitionColumns( - ctx.getStatementContext().getSnapshot(externalTable)))) { + List partitionColumns = getPartitionColumnsForScan(externalTable, scan); + if (CollectionUtils.isEmpty(partitionColumns)) { // non partitioned table, return NOT_PRUNED. // non partition table will be handled in HiveScanNode. return SelectedPartitions.NOT_PRUNED; @@ -85,8 +86,7 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable, Map scanOutput = scan.getOutput() .stream() .collect(Collectors.toMap(slot -> slot.getName().toLowerCase(), Function.identity())); - List partitionSlots = externalTable.getPartitionColumns( - ctx.getStatementContext().getSnapshot(externalTable)) + List partitionSlots = partitionColumns .stream() .map(column -> scanOutput.get(column.getName().toLowerCase())) .collect(Collectors.toList()); @@ -108,4 +108,10 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable, } return new SelectedPartitions(nameToPartitionItem.size(), selectedPartitionItems, true); } + + static List getPartitionColumnsForScan(ExternalTable externalTable, LogicalFileScan scan) { + // The partition map is frozen on the logical relation, so its column list must come from + // the same snapshot rather than the mutable table-scoped compatibility entry. + return externalTable.getPartitionColumns(scan.getRelationSnapshot()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java index 9ec10bed1d8e75..7bc451e48cd609 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java @@ -79,13 +79,18 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public boolean nullable() { - if (child().nullable()) { + return castNullable(child().nullable(), child().getDataType(), targetType); + } + + /** Compute whether a cast can produce null for the given source and target types. */ + public static boolean castNullable(boolean srcNullable, DataType srcType, DataType targetType) { + if (srcNullable) { return true; } // Not allowed cast is forbidden in CheckCast, and all the Propagation Nullable cases are handled above // and the default return false below. // The if branches below only handle 2 cases: always nullable and nullable that may overflow. - DataType childDataType = child().getDataType(); + DataType childDataType = srcType; // StringLike to other type is always nullable. if (childDataType.isStringLikeType() && !targetType.isStringLikeType()) { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java index 3b86d5a474986a..b0ab4725908f5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java @@ -105,8 +105,10 @@ public FunctionSignature customSignature() { ImmutableList.Builder structFields = ImmutableList.builder(); for (int i = 0; i < arity(); i = i + 2) { StringLikeLiteral nameLiteral = (StringLikeLiteral) child(i); + // A named struct has the same value-nullability contract as struct(...); keeping + // the field nullable here would reject safe casts into required target fields. structFields.add(new StructField(nameLiteral.getStringValue(), - children.get(i + 1).getDataType(), true, "")); + children.get(i + 1).getDataType(), children.get(i + 1).nullable(), "")); } return FunctionSignature.ret(new StructType(structFields.build())) .args(children.stream().map(ExpressionTrait::getDataType).toArray(DataType[]::new)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java index 0a4821405a7746..04b36cd8555ae4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.StructField; @@ -31,7 +30,6 @@ import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * struct literal @@ -153,8 +151,16 @@ public static StructType constructStructType(List fieldTypes) { return new StructType(structFields.build()); } + /** + * Infer a struct type from its field expressions. + */ public static StructType computeDataType(List fields) { - List fieldTypes = fields.stream().map(ExpressionTrait::getDataType).collect(Collectors.toList()); - return constructStructType(fieldTypes); + ImmutableList.Builder structFields = ImmutableList.builder(); + for (int i = 0; i < fields.size(); i++) { + Expression field = fields.get(i); + // Preserve literal nullability so a known non-null value can satisfy a required nested field. + structFields.add(new StructField(COL_PREFIX + (i + 1), field.getDataType(), field.nullable(), "")); + } + return new StructType(structFields.build()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergDeleteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergDeleteCommand.java index 370c164d19311e..347c3e654a954e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergDeleteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergDeleteCommand.java @@ -25,7 +25,9 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMergeOperation; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergNereidsUtils; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -55,6 +57,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import org.apache.iceberg.Table; import java.util.List; import java.util.Optional; @@ -155,6 +158,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { IcebergDeleteExecutor deleteExecutor = new IcebergDeleteExecutor( ctx, icebergTable, + ((PhysicalIcebergDeleteSink) physicalSink).getTargetIcebergTable(), label, planner, emptyInsert, @@ -199,6 +203,12 @@ static T executeWithExternalTableBatchModeDisabled( */ private LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuery, IcebergExternalTable icebergTable) { + Optional targetSnapshot = ctx.getStatementContext() + .loadSnapshots(icebergTable, Optional.empty(), Optional.empty()); + Table targetIcebergTable = ((IcebergMvccSnapshot) targetSnapshot.orElseThrow( + () -> new AnalysisException("Iceberg delete target snapshot is not available"))) + .getSnapshotCacheValue().getIcebergTable().orElseThrow( + () -> new AnalysisException("Iceberg delete target metadata is not available")); LogicalPlan queryPlan = buildPositionDeletePlan(ctx, logicalQuery, icebergTable); // Convert output to NamedExpression list @@ -217,7 +227,9 @@ private LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQue LogicalIcebergDeleteSink deleteSink = new LogicalIcebergDeleteSink<>( (IcebergExternalDatabase) icebergTable.getDatabase(), icebergTable, - icebergTable.getBaseSchema(true), // cols + // The sink schema and commit base must stay on the generation bound for the scan. + targetIcebergTable, + icebergTable.getBaseSchema(targetSnapshot, true), // cols outputExprs, // outputExprs deleteCtx, Optional.empty(), // groupExpression diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java index 4d4b9b2014fec6..2caae546af6471 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java @@ -25,9 +25,11 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMergeOperation; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergNereidsUtils; import org.apache.doris.datasource.iceberg.IcebergRowId; import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundRelation; @@ -83,6 +85,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.iceberg.Table; import java.util.ArrayList; import java.util.List; @@ -387,8 +390,8 @@ private List generateFinalProjections(List colNames, return output; } - private LogicalPlan buildMergeProjectPlan(ConnectContext ctx, IcebergExternalTable icebergTable) { - List columns = icebergTable.getBaseSchema(true); + private LogicalPlan buildMergeProjectPlan(ConnectContext ctx, IcebergExternalTable icebergTable, + List columns) { LogicalPlan plan = generateBasePlan(); plan = injectRowIdColumn(plan, icebergTable); @@ -446,7 +449,15 @@ private LogicalPlan buildMergeProjectPlan(ConnectContext ctx, IcebergExternalTab } private LogicalPlan buildMergePlan(ConnectContext ctx, IcebergExternalTable icebergTable) { - LogicalPlan projectPlan = buildMergeProjectPlan(ctx, icebergTable); + Optional targetSnapshot = ctx.getStatementContext() + .loadSnapshots(icebergTable, Optional.empty(), Optional.empty()); + Table targetIcebergTable = ((IcebergMvccSnapshot) targetSnapshot.orElseThrow( + () -> new AnalysisException("Iceberg merge target snapshot is not available"))) + .getSnapshotCacheValue().getIcebergTable().orElseThrow( + () -> new AnalysisException("Iceberg merge target metadata is not available")); + // Bind projections and the sink from the same target generation retained for execution. + List targetSchema = icebergTable.getBaseSchema(targetSnapshot, true); + LogicalPlan projectPlan = buildMergeProjectPlan(ctx, icebergTable, targetSchema); List outputExprs; if (!IcebergNereidsUtils.hasUnboundPlan(projectPlan)) { @@ -462,7 +473,8 @@ private LogicalPlan buildMergePlan(ConnectContext ctx, IcebergExternalTable iceb return new LogicalIcebergMergeSink<>( (IcebergExternalDatabase) icebergTable.getDatabase(), icebergTable, - icebergTable.getBaseSchema(true), + targetIcebergTable, + targetSchema, outputExprs, deleteCtx, true, @@ -492,7 +504,9 @@ private boolean executeMergePlan(ConnectContext ctx, StmtExecutor executor, String label = String.format("iceberg_merge_into_%x_%x", ctx.queryId().hi, ctx.queryId().lo); IcebergMergeExecutor insertExecutor = - new IcebergMergeExecutor(ctx, icebergTable, label, planner, emptyInsert, -1L); + new IcebergMergeExecutor(ctx, icebergTable, + ((PhysicalIcebergMergeSink) physicalSink).getTargetIcebergTable(), + label, planner, emptyInsert, -1L); insertExecutor.setConflictDetectionFilter(conflictFilter); if (insertExecutor.isEmptyInsert()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java index d9d0ab51b58a78..947502c04cbd51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java @@ -25,8 +25,10 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMergeOperation; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergNereidsUtils; import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -60,6 +62,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; +import org.apache.iceberg.Table; import java.util.ArrayList; import java.util.List; @@ -157,7 +160,9 @@ private boolean executeMergePlan(ConnectContext ctx, StmtExecutor executor, String label = String.format("iceberg_update_merge_%x_%x", ctx.queryId().hi, ctx.queryId().lo); IcebergMergeExecutor insertExecutor = - new IcebergMergeExecutor(ctx, icebergTable, label, planner, emptyInsert, -1L); + new IcebergMergeExecutor(ctx, icebergTable, + ((PhysicalIcebergMergeSink) physicalSink).getTargetIcebergTable(), + label, planner, emptyInsert, -1L); insertExecutor.setConflictDetectionFilter(conflictFilter); if (insertExecutor.isEmptyInsert()) { @@ -218,11 +223,19 @@ LogicalPlan buildMergeProjectPlan(ConnectContext ctx, LogicalPlan logicalQuery, private LogicalPlan buildMergePlan(ConnectContext ctx, LogicalPlan logicalQuery, List assignments, IcebergExternalTable icebergTable) { + Optional targetSnapshot = ctx.getStatementContext() + .loadSnapshots(icebergTable, Optional.empty(), Optional.empty()); + Table targetIcebergTable = ((IcebergMvccSnapshot) targetSnapshot.orElseThrow( + () -> new AnalysisException("Iceberg update target snapshot is not available"))) + .getSnapshotCacheValue().getIcebergTable().orElseThrow( + () -> new AnalysisException("Iceberg update target metadata is not available")); + // Use one retained schema for assignment binding, sink output, distribution, and commit. + List targetSchema = icebergTable.getBaseSchema(targetSnapshot, true); String tableName = tableAlias != null ? tableAlias : Util.getTempTableDisplayName(icebergTable.getName()); LogicalPlan queryPlan = buildMergeProjectPlan(ctx, logicalQuery, assignments, - icebergTable.getBaseSchema(true), tableName); + targetSchema, tableName); List outputExprs; if (!IcebergNereidsUtils.hasUnboundPlan(queryPlan)) { @@ -238,7 +251,8 @@ private LogicalPlan buildMergePlan(ConnectContext ctx, LogicalPlan logicalQuery, return new LogicalIcebergMergeSink<>( (IcebergExternalDatabase) icebergTable.getDatabase(), icebergTable, - icebergTable.getBaseSchema(true), + targetIcebergTable, + targetSchema, outputExprs, deleteCtx, false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java index 0bc25dc15519b2..7b6863e9ba58c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java @@ -31,6 +31,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.transaction.TransactionType; +import org.apache.iceberg.Table; import org.apache.iceberg.expressions.Expression; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -51,16 +52,19 @@ public class IcebergDeleteExecutor extends BaseExternalTableInsertExecutor { private static final Logger LOG = LogManager.getLogger(IcebergDeleteExecutor.class); private final NereidsPlanner nereidsPlanner; + private final Table targetIcebergTable; private Optional conflictDetectionFilter = Optional.empty(); private IcebergRewritableDeletePlan rewritableDeletePlan = IcebergRewritableDeletePlan.empty(); public IcebergDeleteExecutor(ConnectContext ctx, IcebergExternalTable table, + Table targetIcebergTable, String labelName, NereidsPlanner planner, boolean emptyInsert, long jobId) { // BaseExternalTableInsertExecutor requires Optional // For DELETE operations, we pass Optional.empty(). super(ctx, table, labelName, planner, Optional.empty(), emptyInsert, jobId); this.nereidsPlanner = planner; + this.targetIcebergTable = targetIcebergTable; } /** Finalize delete sink and attach rewritable delete-file metadata for BE. */ @@ -85,7 +89,8 @@ public void setConflictDetectionFilter(Optional filter) { @Override protected void beforeExec() throws UserException { IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); - transaction.beginDelete((IcebergExternalTable) table); + // DELETE conflict validation must start from the same retained generation scanned by BE. + transaction.beginDelete((IcebergExternalTable) table, targetIcebergTable); transaction.setRewrittenDeleteFilesByReferencedDataFile( rewritableDeletePlan.getDeleteFilesByReferencedDataFile()); if (conflictDetectionFilter.isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java index d5de094860c113..59b0fdd8e4f33a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java @@ -25,6 +25,8 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.transaction.TransactionType; +// Keep third-party imports lexical to preserve the repository's CustomImportOrder invariant. +import org.apache.iceberg.Table; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,21 +37,24 @@ */ public class IcebergInsertExecutor extends BaseExternalTableInsertExecutor { private static final Logger LOG = LogManager.getLogger(IcebergInsertExecutor.class); + private final Table targetIcebergTable; /** * constructor */ public IcebergInsertExecutor(ConnectContext ctx, IcebergExternalTable table, + Table targetIcebergTable, String labelName, NereidsPlanner planner, Optional insertCtx, boolean emptyInsert, long jobId) { super(ctx, table, labelName, planner, insertCtx, emptyInsert, jobId); + this.targetIcebergTable = targetIcebergTable; } @Override protected void beforeExec() throws UserException { IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); - transaction.beginInsert((IcebergExternalTable) table, insertCtx); + transaction.beginInsert((IcebergExternalTable) table, targetIcebergTable, insertCtx); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java index a96dba696eafe2..cf1cfca6deba69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java @@ -32,6 +32,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.transaction.TransactionType; +import org.apache.iceberg.Table; import org.apache.iceberg.expressions.Expression; import java.util.Optional; @@ -41,14 +42,17 @@ */ public class IcebergMergeExecutor extends BaseExternalTableInsertExecutor { private final NereidsPlanner nereidsPlanner; + private final Table targetIcebergTable; private Optional conflictDetectionFilter = Optional.empty(); private IcebergRewritableDeletePlan rewritableDeletePlan = IcebergRewritableDeletePlan.empty(); public IcebergMergeExecutor(ConnectContext ctx, IcebergExternalTable table, + Table targetIcebergTable, String labelName, NereidsPlanner planner, boolean emptyInsert, long jobId) { super(ctx, table, labelName, planner, Optional.empty(), emptyInsert, jobId); this.nereidsPlanner = planner; + this.targetIcebergTable = targetIcebergTable; } /** Finalize merge sink and attach rewritable delete-file metadata for BE. */ @@ -73,7 +77,7 @@ public void setConflictDetectionFilter(Optional filter) { @Override protected void beforeExec() throws UserException { IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); - transaction.beginMerge((IcebergExternalTable) table); + transaction.beginMerge((IcebergExternalTable) table, targetIcebergTable); transaction.setRewrittenDeleteFilesByReferencedDataFile( rewritableDeletePlan.getDeleteFilesByReferencedDataFile()); if (conflictDetectionFilter.isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index ac1464525deaf7..5db7a45f8a959b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -483,7 +483,8 @@ private ExecutorFactory selectInsertExecutorFactory( planner, dataSink, physicalSink, - () -> new IcebergInsertExecutor(ctx, icebergExternalTable, label, planner, + () -> new IcebergInsertExecutor(ctx, icebergExternalTable, + ((PhysicalIcebergTableSink) physicalSink).getTargetIcebergTable(), label, planner, Optional.of(icebergInsertCtx), emptyInsert, jobId ) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index ee98f0ece78bfa..6d4df3ca89f6a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -29,8 +29,10 @@ import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalTable; +import org.apache.doris.datasource.mvcc.MvccTable; import org.apache.doris.foundation.format.FormatOptions; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundBlackholeSink; @@ -285,6 +287,13 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, Optional analyzeContext, Optional insertCtx) { UnboundLogicalSink unboundLogicalSink = (UnboundLogicalSink) plan; + ConnectContext connectContext = ConnectContext.get(); + StatementContext statementContext = analyzeContext.map(CascadesContext::getStatementContext) + .orElseGet(() -> connectContext == null ? null : connectContext.getStatementContext()); + if (table instanceof MvccTable && statementContext != null) { + // Default/generated expressions and sink binding must observe one metadata generation. + statementContext.loadSnapshots(table, Optional.empty(), Optional.empty()); + } if (table instanceof HMSExternalTable) { HMSExternalTable hiveTable = (HMSExternalTable) table; if (hiveTable.isView()) { @@ -377,6 +386,8 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, UnboundInlineTable unboundInlineTable = (UnboundInlineTable) query; ImmutableList.Builder> optimizedRowConstructors = ImmutableList.builderWithExpectedSize(unboundInlineTable.getConstantExprsList().size()); + // Iceberg branch writes follow the shared table schema; historical schemas only apply + // when a branch is read, not when INSERT values are bound. List columns = table.getBaseSchema(false); Map staticPartitions = null; if (unboundLogicalSink instanceof UnboundIcebergTableSink) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index a3397b8c6823f9..6da8d7510e79a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -26,6 +26,8 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccTable; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; @@ -66,6 +68,8 @@ public class LogicalFileScan extends LogicalCatalogRelation implements SupportPr protected final Optional tableSnapshot; protected final Optional scanParams; protected final Optional> cachedOutputs; + protected final Optional> relationSchema; + protected final Optional relationSnapshot; /** * Constructor for LogicalFileScan. @@ -74,12 +78,24 @@ public LogicalFileScan(RelationId id, ExternalTable table, List qualifie Collection operativeSlots, Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional> cachedOutputs) { + this(id, table, qualifier, operativeSlots, tableSample, tableSnapshot, scanParams, cachedOutputs, + MvccUtil.getSnapshotFromContext(table)); + } + + /** + * Constructor for a relation whose concrete snapshot was resolved during binding. + */ + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + Collection operativeSlots, + Optional tableSample, Optional tableSnapshot, + Optional scanParams, Optional> cachedOutputs, + Optional relationSnapshot) { this(id, table, qualifier, - initialSelectedPartitions(table, scanParams), + initialSelectedPartitions(table, scanParams, relationSnapshot), operativeSlots, ImmutableList.of(), tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.empty(), - cachedOutputs); + cachedOutputs, captureRelationSchema(table, scanParams, relationSnapshot), relationSnapshot); } /** @@ -91,6 +107,31 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali Optional tableSnapshot, Optional scanParams, Optional groupExpression, Optional logicalProperties, Optional> cachedSlots) { + this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, + scanParams, groupExpression, logicalProperties, cachedSlots, Optional.empty()); + } + + /** + * Constructor for LogicalFileScan. + */ + protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + SelectedPartitions selectedPartitions, Collection operativeSlots, + List virtualColumns, Optional tableSample, + Optional tableSnapshot, Optional scanParams, + Optional groupExpression, Optional logicalProperties, + Optional> cachedSlots, Optional> relationSchema) { + this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, + scanParams, groupExpression, logicalProperties, cachedSlots, relationSchema, + MvccUtil.getSnapshotFromContext(table)); + } + + protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + SelectedPartitions selectedPartitions, Collection operativeSlots, + List virtualColumns, Optional tableSample, + Optional tableSnapshot, Optional scanParams, + Optional groupExpression, Optional logicalProperties, + Optional> cachedSlots, Optional> relationSchema, + Optional relationSnapshot) { super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, operativeSlots, virtualColumns, groupExpression, logicalProperties); this.selectedPartitions = selectedPartitions; @@ -98,17 +139,46 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali this.tableSnapshot = tableSnapshot; this.scanParams = scanParams; this.cachedOutputs = cachedSlots; + this.relationSchema = relationSchema; + this.relationSnapshot = relationSnapshot; } private static SelectedPartitions initialSelectedPartitions( - ExternalTable table, Optional scanParams) { + ExternalTable table, Optional scanParams, + Optional relationSnapshot) { if ((table instanceof PaimonExternalTable || table instanceof PaimonSysExternalTable) && scanParams.isPresent() && scanParams.get().isOptions()) { // A relation-scoped historical snapshot cannot reuse partitions cached for the // statement-level latest snapshot; Paimon will prune its selected snapshot instead. return SelectedPartitions.NOT_PRUNED; } - return table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)); + return table.initSelectedPartitions(relationSnapshot); + } + + private static Optional> captureRelationSchema( + ExternalTable table, Optional scanParams, + Optional relationSnapshot) { + if (scanParams.isPresent() && scanParams.get().isOptions()) { + if (table instanceof PaimonExternalTable) { + return Optional.of(ImmutableList.copyOf( + ((PaimonExternalTable) table).getFullSchema(scanParams.get()))); + } + if (table instanceof PaimonSysExternalTable) { + return Optional.of(ImmutableList.copyOf( + ((PaimonSysExternalTable) table).getFullSchema(scanParams.get()))); + } + } + return captureRelationSchema(table, relationSnapshot); + } + + protected static Optional> captureRelationSchema( + ExternalTable table, Optional relationSnapshot) { + if (!(table instanceof MvccTable)) { + return Optional.empty(); + } + // Pin columns while this relation's snapshot is current, but create slots lazily to + // preserve statement-wide ExprId allocation order used by materialized-view rewrites. + return Optional.of(ImmutableList.copyOf(table.getFullSchema(relationSnapshot))); } public SelectedPartitions getSelectedPartitions() { @@ -127,6 +197,10 @@ public Optional getScanParams() { return scanParams; } + public Optional getRelationSnapshot() { + return relationSnapshot; + } + @Override public ExternalTable getTable() { Preconditions.checkArgument(table instanceof ExternalTable, @@ -148,7 +222,8 @@ public String toString() { public LogicalFileScan withGroupExpression(Optional groupExpression) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, - scanParams, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs); + scanParams, groupExpression, Optional.of(getLogicalProperties()), + cachedOutputs, relationSchema, relationSnapshot); } @Override @@ -156,20 +231,23 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, - scanParams, groupExpression, logicalProperties, cachedOutputs); + scanParams, groupExpression, logicalProperties, cachedOutputs, + relationSchema, relationSnapshot); } public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, - scanParams, Optional.empty(), Optional.of(getLogicalProperties()), cachedOutputs); + scanParams, Optional.empty(), Optional.of(getLogicalProperties()), + cachedOutputs, relationSchema, relationSnapshot); } @Override public LogicalFileScan withRelationId(RelationId relationId) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, - scanParams, Optional.empty(), Optional.empty(), cachedOutputs); + scanParams, Optional.empty(), Optional.empty(), cachedOutputs, + relationSchema, relationSnapshot); } @Override @@ -191,7 +269,14 @@ protected boolean hasSameScanState(LogicalCatalogRelation other) { return Objects.equals(selectedPartitions, that.selectedPartitions) && Objects.equals(tableSample, that.tableSample) && hasSameSnapshot(tableSnapshot, that.tableSnapshot) - && hasSameScanParams(scanParams, that.scanParams); + && hasSameScanParams(scanParams, that.scanParams) + && hasSameResolvedSnapshot(relationSnapshot, that.relationSnapshot); + } + + private static boolean hasSameResolvedSnapshot( + Optional left, Optional right) { + return left.isPresent() == right.isPresent() + && (!left.isPresent() || left.get().isSameSnapshot(right.get())); } @Override @@ -200,6 +285,10 @@ public List computeOutput() { return cachedOutputs.get(); } + if (relationSchema.isPresent()) { + return computeOutput(relationSchema.get()); + } + if (table instanceof IcebergExternalTable) { // iceberg v3 need append row lineage columns return computeIcebergOutput(); @@ -217,8 +306,7 @@ public List computeOutput() { private List computeOutput(List schema) { IdGenerator exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator(); Builder slots = ImmutableList.builder(); - schema - .stream() + schema.stream() .map(col -> SlotReference.fromColumn(exprIdGenerator.getNextId(), table, col, qualified())) .forEach(slots::add); // add virtual slots @@ -341,13 +429,15 @@ public int hashCode() { public LogicalFileScan withOperativeSlots(Collection operativeSlots) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, - scanParams, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs); + scanParams, groupExpression, Optional.of(getLogicalProperties()), + cachedOutputs, relationSchema, relationSnapshot); } public LogicalFileScan withCachedOutput(List cachedOutputs) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, - scanParams, groupExpression, Optional.empty(), Optional.of(cachedOutputs)); + scanParams, groupExpression, Optional.empty(), Optional.of(cachedOutputs), + relationSchema, relationSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java index 18ad58f9388528..c66e01e1587223 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; +import org.apache.doris.catalog.Column; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; @@ -26,6 +27,8 @@ import org.apache.doris.datasource.hudi.source.EmptyIncrementalRelation; import org.apache.doris.datasource.hudi.source.IncrementalRelation; import org.apache.doris.datasource.hudi.source.MORIncrementalRelation; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; @@ -79,8 +82,25 @@ protected LogicalHudiScan(RelationId id, ExternalTable table, List quali Optional groupExpression, Optional logicalProperties, Optional> cachedOutputs) { + this(id, table, qualifier, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, + operativeSlots, virtualColumns, groupExpression, logicalProperties, cachedOutputs, + Optional.empty(), MvccUtil.getSnapshotFromContext(table)); + } + + protected LogicalHudiScan(RelationId id, ExternalTable table, List qualifier, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Optional scanParams, Optional incrementalRelation, + Collection operativeSlots, + List virtualColumns, + Optional groupExpression, + Optional logicalProperties, + Optional> cachedOutputs, + Optional> relationSchema, + Optional relationSnapshot) { super(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, - tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs); + tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs, + relationSchema, relationSnapshot); Objects.requireNonNull(scanParams, "scanParams should not null"); Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null"); this.incrementalRelation = incrementalRelation; @@ -95,6 +115,19 @@ public LogicalHudiScan(RelationId id, ExternalTable table, List qualifie Optional.empty(), Optional.empty(), cachedOutputs); } + /** + * Constructor for a Hudi relation whose concrete snapshot was resolved during binding. + */ + public LogicalHudiScan(RelationId id, ExternalTable table, List qualifier, + Collection operativeSlots, Optional scanParams, + Optional tableSample, Optional tableSnapshot, + Optional> cachedOutputs, Optional relationSnapshot) { + this(id, table, qualifier, table.initSelectedPartitions(relationSnapshot), + tableSample, tableSnapshot, scanParams, Optional.empty(), operativeSlots, ImmutableList.of(), + Optional.empty(), Optional.empty(), cachedOutputs, + captureRelationSchema(table, relationSnapshot), relationSnapshot); + } + public Optional getScanParams() { return scanParams; } @@ -152,7 +185,8 @@ public String toString() { public LogicalHudiScan withGroupExpression(Optional groupExpression) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, - operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs); + operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), + cachedOutputs, relationSchema, relationSnapshot); } @Override @@ -160,20 +194,23 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, - operativeSlots, virtualColumns, groupExpression, logicalProperties, cachedOutputs); + operativeSlots, virtualColumns, groupExpression, logicalProperties, + cachedOutputs, relationSchema, relationSnapshot); } public LogicalHudiScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, - operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs); + operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), + cachedOutputs, relationSchema, relationSnapshot); } @Override public LogicalHudiScan withRelationId(RelationId relationId) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, - operativeSlots, virtualColumns, groupExpression, Optional.empty(), cachedOutputs); + operativeSlots, virtualColumns, groupExpression, Optional.empty(), + cachedOutputs, relationSchema, relationSnapshot); } @Override @@ -182,18 +219,19 @@ public R accept(PlanVisitor visitor, C context) { } @Override - public LogicalFileScan withOperativeSlots(Collection operativeSlots) { + public LogicalHudiScan withOperativeSlots(Collection operativeSlots) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, - selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, - operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs); + selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, + operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), + cachedOutputs, relationSchema, relationSnapshot); } @Override - public LogicalFileScan withCachedOutput(List cachedOutputs) { + public LogicalHudiScan withCachedOutput(List cachedOutputs) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, - selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, - operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), - Optional.of(cachedOutputs)); + selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, + operativeSlots, virtualColumns, groupExpression, Optional.empty(), + Optional.of(cachedOutputs), relationSchema, relationSnapshot); } /** @@ -245,7 +283,8 @@ public LogicalHudiScan withScanParams(HMSExternalTable table, Optional extends LogicalTa implements Sink, PropagateFuncDeps { private final IcebergExternalDatabase database; private final IcebergExternalTable targetTable; + private final Table targetIcebergTable; private final DeleteCommandContext deleteContext; /** @@ -53,6 +55,7 @@ public class LogicalIcebergDeleteSink extends LogicalTa */ public LogicalIcebergDeleteSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, DeleteCommandContext deleteContext, @@ -62,6 +65,8 @@ public LogicalIcebergDeleteSink(IcebergExternalDatabase database, super(PlanType.LOGICAL_ICEBERG_DELETE_SINK, outputExprs, groupExpression, logicalProperties, cols, child); this.database = Objects.requireNonNull(database, "database != null in LogicalIcebergDeleteSink"); this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalIcebergDeleteSink"); + this.targetIcebergTable = Objects.requireNonNull( + targetIcebergTable, "targetIcebergTable != null in LogicalIcebergDeleteSink"); this.deleteContext = Objects.requireNonNull(deleteContext, "deleteContext != null in LogicalIcebergDeleteSink"); } @@ -69,19 +74,19 @@ public Plan withChildAndUpdateOutput(Plan child) { List output = child.getOutput().stream() .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); - return new LogicalIcebergDeleteSink<>(database, targetTable, cols, output, + return new LogicalIcebergDeleteSink<>(database, targetTable, targetIcebergTable, cols, output, deleteContext, Optional.empty(), Optional.empty(), child); } @Override public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "LogicalIcebergDeleteSink only accepts one child"); - return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergDeleteSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, Optional.empty(), Optional.empty(), children.get(0)); } public LogicalIcebergDeleteSink withOutputExprs(List outputExprs) { - return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergDeleteSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, Optional.empty(), Optional.empty(), child()); } @@ -93,6 +98,10 @@ public IcebergExternalTable getTargetTable() { return targetTable; } + public Table getTargetIcebergTable() { + return targetIcebergTable; + } + public DeleteCommandContext getDeleteContext() { return deleteContext; } @@ -111,13 +120,14 @@ public boolean equals(Object o) { LogicalIcebergDeleteSink that = (LogicalIcebergDeleteSink) o; return Objects.equals(database, that.database) && Objects.equals(targetTable, that.targetTable) + && Objects.equals(targetIcebergTable, that.targetIcebergTable) && Objects.equals(deleteContext, that.deleteContext) && Objects.equals(cols, that.cols); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), database, targetTable, cols, deleteContext); + return Objects.hash(super.hashCode(), database, targetTable, targetIcebergTable, cols, deleteContext); } @Override @@ -138,14 +148,14 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergDeleteSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergDeleteSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, groupExpression, logicalProperties, children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java index c8198c27dcce22..f0f4a24196ce62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java @@ -33,6 +33,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import org.apache.iceberg.Table; import java.util.List; import java.util.Objects; @@ -46,6 +47,7 @@ public class LogicalIcebergMergeSink extends LogicalTab implements Sink, PropagateFuncDeps { private final IcebergExternalDatabase database; private final IcebergExternalTable targetTable; + private final Table targetIcebergTable; private final DeleteCommandContext deleteContext; private final boolean requireMergeCardinalityCheck; @@ -54,6 +56,7 @@ public class LogicalIcebergMergeSink extends LogicalTab */ public LogicalIcebergMergeSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, DeleteCommandContext deleteContext, @@ -64,6 +67,8 @@ public LogicalIcebergMergeSink(IcebergExternalDatabase database, super(PlanType.LOGICAL_ICEBERG_MERGE_SINK, outputExprs, groupExpression, logicalProperties, cols, child); this.database = Objects.requireNonNull(database, "database != null in LogicalIcebergMergeSink"); this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalIcebergMergeSink"); + this.targetIcebergTable = Objects.requireNonNull( + targetIcebergTable, "targetIcebergTable != null in LogicalIcebergMergeSink"); this.deleteContext = Objects.requireNonNull(deleteContext, "deleteContext != null in LogicalIcebergMergeSink"); this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; } @@ -72,19 +77,19 @@ public Plan withChildAndUpdateOutput(Plan child) { List output = child.getOutput().stream() .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); - return new LogicalIcebergMergeSink<>(database, targetTable, cols, output, + return new LogicalIcebergMergeSink<>(database, targetTable, targetIcebergTable, cols, output, deleteContext, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), child); } @Override public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "LogicalIcebergMergeSink only accepts one child"); - return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergMergeSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), children.get(0)); } public LogicalIcebergMergeSink withOutputExprs(List outputExprs) { - return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergMergeSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), child()); } @@ -96,6 +101,10 @@ public IcebergExternalTable getTargetTable() { return targetTable; } + public Table getTargetIcebergTable() { + return targetIcebergTable; + } + public DeleteCommandContext getDeleteContext() { return deleteContext; } @@ -118,6 +127,7 @@ public boolean equals(Object o) { LogicalIcebergMergeSink that = (LogicalIcebergMergeSink) o; return Objects.equals(database, that.database) && Objects.equals(targetTable, that.targetTable) + && Objects.equals(targetIcebergTable, that.targetIcebergTable) && Objects.equals(deleteContext, that.deleteContext) && requireMergeCardinalityCheck == that.requireMergeCardinalityCheck && Objects.equals(cols, that.cols); @@ -125,7 +135,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), database, targetTable, cols, deleteContext, + return Objects.hash(super.hashCode(), database, targetTable, targetIcebergTable, cols, deleteContext, requireMergeCardinalityCheck); } @@ -147,7 +157,7 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergMergeSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, groupExpression, Optional.of(getLogicalProperties()), child()); } @@ -155,7 +165,7 @@ public Plan withGroupExpression(Optional groupExpression) { @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergMergeSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, groupExpression, logicalProperties, children.get(0)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java index 91398ddc5ac78c..d9f549f94c095c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java @@ -33,6 +33,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import org.apache.iceberg.Table; import java.util.List; import java.util.Objects; @@ -46,6 +47,7 @@ public class LogicalIcebergTableSink extends LogicalTab // bound data sink private final IcebergExternalDatabase database; private final IcebergExternalTable targetTable; + private final Table targetIcebergTable; private final DMLCommandType dmlCommandType; /** @@ -53,6 +55,7 @@ public class LogicalIcebergTableSink extends LogicalTab */ public LogicalIcebergTableSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, DMLCommandType dmlCommandType, @@ -62,6 +65,8 @@ public LogicalIcebergTableSink(IcebergExternalDatabase database, super(PlanType.LOGICAL_ICEBERG_TABLE_SINK, outputExprs, groupExpression, logicalProperties, cols, child); this.database = Objects.requireNonNull(database, "database != null in LogicalIcebergTableSink"); this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalIcebergTableSink"); + this.targetIcebergTable = Objects.requireNonNull( + targetIcebergTable, "targetIcebergTable != null in LogicalIcebergTableSink"); this.dmlCommandType = dmlCommandType; } @@ -69,19 +74,19 @@ public Plan withChildAndUpdateOutput(Plan child) { List output = child.getOutput().stream() .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); - return new LogicalIcebergTableSink<>(database, targetTable, cols, output, + return new LogicalIcebergTableSink<>(database, targetTable, targetIcebergTable, cols, output, dmlCommandType, Optional.empty(), Optional.empty(), child); } @Override public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "LogicalIcebergTableSink only accepts one child"); - return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergTableSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, dmlCommandType, Optional.empty(), Optional.empty(), children.get(0)); } public LogicalIcebergTableSink withOutputExprs(List outputExprs) { - return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergTableSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, dmlCommandType, Optional.empty(), Optional.empty(), child()); } @@ -93,6 +98,10 @@ public IcebergExternalTable getTargetTable() { return targetTable; } + public Table getTargetIcebergTable() { + return targetIcebergTable; + } + public DMLCommandType getDmlCommandType() { return dmlCommandType; } @@ -111,12 +120,14 @@ public boolean equals(Object o) { LogicalIcebergTableSink that = (LogicalIcebergTableSink) o; return dmlCommandType == that.dmlCommandType && Objects.equals(database, that.database) - && Objects.equals(targetTable, that.targetTable) && Objects.equals(cols, that.cols); + && Objects.equals(targetTable, that.targetTable) + && Objects.equals(targetIcebergTable, that.targetIcebergTable) + && Objects.equals(cols, that.cols); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), database, targetTable, cols, dmlCommandType); + return Objects.hash(super.hashCode(), database, targetTable, targetIcebergTable, cols, dmlCommandType); } @Override @@ -137,14 +148,14 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergTableSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + return new LogicalIcebergTableSink<>(database, targetTable, targetIcebergTable, cols, outputExprs, dmlCommandType, groupExpression, logicalProperties, children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 8d8642231eae53..37824a0f40c8e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -276,9 +276,13 @@ private static DataType getAssignmentCompatibleTypeLegacy(DataType left, DataTyp } ImmutableList.Builder commonFields = ImmutableList.builder(); for (int i = 0; i < leftFields.size(); i++) { - boolean nullable = leftFields.get(i).isNullable() || rightFields.get(i).isNullable(); DataType commonType = getAssignmentCompatibleType( leftFields.get(i).getDataType(), rightFields.get(i).getDataType()); + // Legacy set operations need a nullable common child whenever coercing either + // input can itself produce NULL, even if both source fields are declared required. + boolean nullable = leftFields.get(i).isNullable() || rightFields.get(i).isNullable() + || Cast.castNullable(false, leftFields.get(i).getDataType(), commonType) + || Cast.castNullable(false, rightFields.get(i).getDataType(), commonType); StructField commonField = leftFields.get(i).withDataTypeAndNullable(commonType, nullable); commonFields.add(commonField); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index 87e311c6375aab..de63f29baefbce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; @@ -48,6 +49,7 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { protected final Optional tableSample; protected final Optional tableSnapshot; protected final Optional scanParams; + protected final Optional relationSnapshot; /** * Constructor for PhysicalFileScan. @@ -59,8 +61,20 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi Optional tableSnapshot, Collection operativeSlots, Optional scanParams) { + this(id, table, qualifier, distributionSpec, groupExpression, logicalProperties, selectedPartitions, + tableSample, tableSnapshot, operativeSlots, scanParams, Optional.empty()); + } + + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Collection operativeSlots, + Optional scanParams, Optional relationSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, - logicalProperties, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams); + logicalProperties, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, + relationSnapshot); } /** @@ -72,9 +86,21 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Collection operativeSlots, Optional scanParams) { + this(id, table, qualifier, distributionSpec, groupExpression, logicalProperties, physicalProperties, + statistics, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, + Optional.empty()); + } + + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot, + Collection operativeSlots, Optional scanParams, + Optional relationSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams); + operativeSlots, scanParams, relationSnapshot); } /** @@ -87,12 +113,24 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li Optional tableSnapshot, Collection operativeSlots, Optional scanParams) { + this(id, type, table, qualifier, distributionSpec, groupExpression, logicalProperties, selectedPartitions, + tableSample, tableSnapshot, operativeSlots, scanParams, Optional.empty()); + } + + protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Collection operativeSlots, + Optional scanParams, Optional relationSnapshot) { super(id, type, table, qualifier, groupExpression, logicalProperties, operativeSlots); this.distributionSpec = distributionSpec; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; this.scanParams = scanParams; + this.relationSnapshot = relationSnapshot; } protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, @@ -101,6 +139,18 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Collection operativeSlots, Optional scanParams) { + this(id, type, table, qualifier, distributionSpec, groupExpression, logicalProperties, physicalProperties, + statistics, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, + Optional.empty()); + } + + protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot, + Collection operativeSlots, Optional scanParams, + Optional relationSnapshot) { super(id, type, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics, operativeSlots); this.distributionSpec = distributionSpec; @@ -108,6 +158,7 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; this.scanParams = scanParams; + this.relationSnapshot = relationSnapshot; } public DistributionSpec getDistributionSpec() { @@ -130,6 +181,10 @@ public Optional getScanParams() { return scanParams; } + public Optional getRelationSnapshot() { + return relationSnapshot; + } + @Override public String toString() { String rfV2 = ""; @@ -155,7 +210,7 @@ public R accept(PlanVisitor visitor, C context) { public PhysicalFileScan withGroupExpression(Optional groupExpression) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams); + operativeSlots, scanParams, relationSnapshot); } @Override @@ -163,7 +218,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams); + operativeSlots, scanParams, relationSnapshot); } @Override @@ -177,7 +232,7 @@ public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physic return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams); + operativeSlots, scanParams, relationSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java index a5ed2f28865d6b..fa345b930a4748 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hudi.source.IncrementalRelation; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; @@ -58,8 +59,19 @@ public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifi Optional tableSnapshot, Optional scanParams, Optional incrementalRelation, Collection operativeSlots) { + this(id, table, qualifier, distributionSpec, groupExpression, logicalProperties, selectedPartitions, + tableSample, tableSnapshot, scanParams, incrementalRelation, operativeSlots, Optional.empty()); + } + + public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Optional scanParams, Optional incrementalRelation, + Collection operativeSlots, Optional relationSnapshot) { super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, - selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams); + selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, relationSnapshot); Objects.requireNonNull(scanParams, "scanParams should not null"); Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null"); this.incrementalRelation = incrementalRelation; @@ -75,9 +87,21 @@ public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifi Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional incrementalRelation, Collection operativeSlots) { + this(id, table, qualifier, distributionSpec, groupExpression, logicalProperties, physicalProperties, + statistics, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation, + operativeSlots, Optional.empty()); + } + + public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot, + Optional scanParams, Optional incrementalRelation, + Collection operativeSlots, Optional relationSnapshot) { super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams); + operativeSlots, scanParams, relationSnapshot); this.incrementalRelation = incrementalRelation; } @@ -93,7 +117,7 @@ public Optional getIncrementalRelation() { public PhysicalHudiScan withGroupExpression(Optional groupExpression) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot, - scanParams, incrementalRelation, operativeSlots); + scanParams, incrementalRelation, operativeSlots, relationSnapshot); } @Override @@ -101,7 +125,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot, - scanParams, incrementalRelation, operativeSlots); + scanParams, incrementalRelation, operativeSlots, relationSnapshot); } @Override @@ -110,7 +134,7 @@ public PhysicalHudiScan withPhysicalPropertiesAndStats(PhysicalProperties physic return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - scanParams, incrementalRelation, operativeSlots); + scanParams, incrementalRelation, operativeSlots, relationSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergDeleteSink.java index 67192dbb4fb14e..3247caee33ab6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergDeleteSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergDeleteSink.java @@ -36,6 +36,7 @@ import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; +import org.apache.iceberg.Table; import java.util.List; import java.util.Objects; @@ -47,19 +48,22 @@ */ public class PhysicalIcebergDeleteSink extends PhysicalBaseExternalTableSink { private final DeleteCommandContext deleteContext; + private final Table targetIcebergTable; /** * Constructor */ public PhysicalIcebergDeleteSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, DeleteCommandContext deleteContext, Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(database, targetTable, cols, outputExprs, deleteContext, groupExpression, logicalProperties, + this(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, + groupExpression, logicalProperties, PhysicalProperties.GATHER, null, child); } @@ -68,6 +72,7 @@ public PhysicalIcebergDeleteSink(IcebergExternalDatabase database, */ public PhysicalIcebergDeleteSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, DeleteCommandContext deleteContext, @@ -80,17 +85,23 @@ public PhysicalIcebergDeleteSink(IcebergExternalDatabase database, logicalProperties, physicalProperties, statistics, child); this.deleteContext = Objects.requireNonNull( deleteContext, "deleteContext != null in PhysicalIcebergDeleteSink"); + this.targetIcebergTable = Objects.requireNonNull( + targetIcebergTable, "targetIcebergTable != null in PhysicalIcebergDeleteSink"); } public DeleteCommandContext getDeleteContext() { return deleteContext; } + public Table getTargetIcebergTable() { + return targetIcebergTable; + } + @Override public Plan withChildren(List children) { return new PhysicalIcebergDeleteSink<>( (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, - cols, outputExprs, deleteContext, groupExpression, + targetIcebergTable, cols, outputExprs, deleteContext, groupExpression, getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @@ -102,23 +113,26 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return new PhysicalIcebergDeleteSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, groupExpression, getLogicalProperties(), child()); + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, deleteContext, groupExpression, + getLogicalProperties(), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalIcebergDeleteSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, groupExpression, logicalProperties.get(), children.get(0)); + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, deleteContext, groupExpression, + logicalProperties.get(), children.get(0)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalIcebergDeleteSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, deleteContext, groupExpression, getLogicalProperties(), + physicalProperties, statistics, child()); } @Override @@ -133,12 +147,13 @@ public boolean equals(Object o) { return false; } PhysicalIcebergDeleteSink that = (PhysicalIcebergDeleteSink) o; - return Objects.equals(deleteContext, that.deleteContext); + return Objects.equals(deleteContext, that.deleteContext) + && Objects.equals(targetIcebergTable, that.targetIcebergTable); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), deleteContext); + return Objects.hash(super.hashCode(), deleteContext, targetIcebergTable); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java index e182c5adc06302..fed451b2b8595f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java @@ -56,12 +56,14 @@ public class PhysicalIcebergMergeSink extends PhysicalBaseExternalTableSink { private final DeleteCommandContext deleteContext; private final boolean requireMergeCardinalityCheck; + private final Table targetIcebergTable; /** * Constructor */ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, DeleteCommandContext deleteContext, @@ -69,7 +71,8 @@ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(database, targetTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, + this(database, targetTable, targetIcebergTable, cols, outputExprs, deleteContext, + requireMergeCardinalityCheck, groupExpression, logicalProperties, PhysicalProperties.GATHER, null, child); } @@ -79,6 +82,7 @@ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, */ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, DeleteCommandContext deleteContext, @@ -93,6 +97,8 @@ public PhysicalIcebergMergeSink(IcebergExternalDatabase database, this.deleteContext = Objects.requireNonNull( deleteContext, "deleteContext != null in PhysicalIcebergMergeSink"); this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; + this.targetIcebergTable = Objects.requireNonNull( + targetIcebergTable, "targetIcebergTable != null in PhysicalIcebergMergeSink"); } public DeleteCommandContext getDeleteContext() { @@ -103,11 +109,16 @@ public boolean isRequireMergeCardinalityCheck() { return requireMergeCardinalityCheck; } + public Table getTargetIcebergTable() { + return targetIcebergTable; + } + @Override public Plan withChildren(List children) { return new PhysicalIcebergMergeSink<>( (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, - cols, outputExprs, deleteContext, requireMergeCardinalityCheck, groupExpression, + targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, + groupExpression, getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @@ -119,8 +130,8 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return new PhysicalIcebergMergeSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, requireMergeCardinalityCheck, + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, groupExpression, getLogicalProperties(), child()); } @@ -128,17 +139,18 @@ public Plan withGroupExpression(Optional groupExpression) { public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalIcebergMergeSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, requireMergeCardinalityCheck, + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, groupExpression, logicalProperties.get(), children.get(0)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalIcebergMergeSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - deleteContext, requireMergeCardinalityCheck, - groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, deleteContext, requireMergeCardinalityCheck, + groupExpression, getLogicalProperties(), + physicalProperties, statistics, child()); } @Override @@ -154,12 +166,13 @@ public boolean equals(Object o) { } PhysicalIcebergMergeSink that = (PhysicalIcebergMergeSink) o; return Objects.equals(deleteContext, that.deleteContext) - && requireMergeCardinalityCheck == that.requireMergeCardinalityCheck; + && requireMergeCardinalityCheck == that.requireMergeCardinalityCheck + && Objects.equals(targetIcebergTable, that.targetIcebergTable); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), deleteContext, requireMergeCardinalityCheck); + return Objects.hash(super.hashCode(), deleteContext, targetIcebergTable, requireMergeCardinalityCheck); } /** @@ -205,14 +218,15 @@ public PhysicalProperties getRequirePhysicalProperties() { List insertPartitionExprIds = new ArrayList<>(); List insertPartitionFields = new ArrayList<>(); Integer partitionSpecId = null; - List partitionColumns = ((IcebergExternalTable) targetTable).getPartitionColumns(Optional.empty()); + // Distribution and writer serialization must read the same retained spec/schema. + List partitionColumns = getRetainedPartitionColumns(); Map columnExprIdMap = buildColumnExprIdMap(outputSlots, nameToExprId); boolean insertExprsOk = false; if (!partitionColumns.isEmpty()) { insertExprsOk = buildInsertPartitionExprIds(insertPartitionExprIds, partitionColumns, columnExprIdMap); } InsertPartitionFieldResult fieldResult = buildInsertPartitionFields( - insertPartitionFields, (IcebergExternalTable) targetTable, columnExprIdMap); + insertPartitionFields, targetIcebergTable, columnExprIdMap); boolean insertFieldsOk = fieldResult.success; boolean hasNonIdentity = fieldResult.hasNonIdentity; if (insertFieldsOk) { @@ -288,12 +302,8 @@ private List getDataSlots(List outputSlots) { private InsertPartitionFieldResult buildInsertPartitionFields( List insertPartitionFields, - IcebergExternalTable icebergTable, + Table table, Map columnExprIdMap) { - Table table = icebergTable.getIcebergTable(); - if (table == null) { - return new InsertPartitionFieldResult(false, false, null); - } PartitionSpec spec = table.spec(); if (spec == null || !spec.isPartitioned()) { return new InsertPartitionFieldResult(false, false, null); @@ -331,6 +341,27 @@ private InsertPartitionFieldResult buildInsertPartitionFields( return new InsertPartitionFieldResult(true, hasNonIdentity, spec.specId()); } + private List getRetainedPartitionColumns() { + Map columnsByName = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (Column column : cols) { + columnsByName.put(column.getName(), column); + } + List partitionColumns = new ArrayList<>(); + Schema schema = targetIcebergTable.schema(); + for (PartitionField field : targetIcebergTable.spec().fields()) { + // Transformed fields are encoded through insertPartitionFields; treating their source + // columns as identity keys would route rows by a different partitioning invariant. + if (!field.transform().isIdentity()) { + continue; + } + Types.NestedField sourceField = schema.findField(field.sourceId()); + if (sourceField != null && columnsByName.containsKey(sourceField.name())) { + partitionColumns.add(columnsByName.get(sourceField.name())); + } + } + return partitionColumns; + } + private Integer parseTransformParam(String transform) { int start = transform.indexOf('['); int end = transform.indexOf(']'); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java index 12839e934935d8..cfcd4ac0a73299 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java @@ -32,26 +32,33 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.Statistics; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.Table; +import org.apache.iceberg.types.Types; + import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; /** physical iceberg sink */ public class PhysicalIcebergTableSink extends PhysicalBaseExternalTableSink { + private final Table targetIcebergTable; /** * constructor */ public PhysicalIcebergTableSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(database, targetTable, cols, outputExprs, groupExpression, logicalProperties, + this(database, targetTable, targetIcebergTable, cols, outputExprs, groupExpression, logicalProperties, PhysicalProperties.GATHER, null, child); } @@ -60,6 +67,7 @@ public PhysicalIcebergTableSink(IcebergExternalDatabase database, */ public PhysicalIcebergTableSink(IcebergExternalDatabase database, IcebergExternalTable targetTable, + Table targetIcebergTable, List cols, List outputExprs, Optional groupExpression, @@ -69,13 +77,15 @@ public PhysicalIcebergTableSink(IcebergExternalDatabase database, CHILD_TYPE child) { super(PlanType.PHYSICAL_ICEBERG_TABLE_SINK, database, targetTable, cols, outputExprs, groupExpression, logicalProperties, physicalProperties, statistics, child); + this.targetIcebergTable = Objects.requireNonNull( + targetIcebergTable, "targetIcebergTable != null in PhysicalIcebergTableSink"); } @Override public Plan withChildren(List children) { return new PhysicalIcebergTableSink<>( (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, - cols, outputExprs, groupExpression, + targetIcebergTable, cols, outputExprs, groupExpression, getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @@ -87,23 +97,28 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return new PhysicalIcebergTableSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - groupExpression, getLogicalProperties(), child()); + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, groupExpression, getLogicalProperties(), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalIcebergTableSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - groupExpression, logicalProperties.get(), children.get(0)); + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, groupExpression, logicalProperties.get(), children.get(0)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalIcebergTableSink<>( - (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, - groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + targetIcebergTable, cols, outputExprs, groupExpression, getLogicalProperties(), + physicalProperties, statistics, child()); + } + + public Table getTargetIcebergTable() { + return targetIcebergTable; } /** @@ -120,13 +135,19 @@ public PhysicalProperties getRequirePhysicalProperties() { return PhysicalProperties.GATHER; } - Set partitionNames = targetTable.getPartitionNames(); + Set partitionNames = new java.util.TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (PartitionField field : targetIcebergTable.spec().fields()) { + Types.NestedField sourceField = targetIcebergTable.schema().findField(field.sourceId()); + if (sourceField != null) { + partitionNames.add(sourceField.name()); + } + } if (!partitionNames.isEmpty()) { List columnIdx = new ArrayList<>(); - List fullSchema = targetTable.getFullSchema(); - for (int i = 0; i < fullSchema.size(); i++) { - Column column = fullSchema.get(i); - if (partitionNames.contains(column.getName())) { + // Sink columns are bound from targetIcebergTable; using refreshable table metadata + // here could shuffle rows with a different partition spec than the writer serializes. + for (int i = 0; i < child().getOutput().size(); i++) { + if (partitionNames.contains(child().getOutput().get(i).getName())) { columnIdx.add(i); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java index b062048570952c..4c0befccf97005 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java @@ -42,7 +42,7 @@ public PhysicalLazyMaterializeFileScan(PhysicalFileScan scan, SlotReference rowI Optional.empty(), null, null, scan.getStats(), scan.selectedPartitions, scan.getTableSample(), scan.getTableSnapshot(), scan.getOperativeSlots(), - scan.getScanParams()); + scan.getScanParams(), scan.getRelationSnapshot()); this.scan = scan; this.rowId = rowId; this.lazySlots = lazySlots; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 673631bd16ea2e..a10eabe2a736ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -208,7 +208,10 @@ public static Optional implicitCast(DataType input, DataType expected) Optional newDataType = implicitCast(inputFields.get(i).getDataType(), expectedFields.get(i).getDataType()); if (newDataType.isPresent()) { - newFields.add(inputFields.get(i).withDataType(newDataType.get())); + // The struct layout must also admit NULLs introduced by a fallible child cast. + boolean nullable = inputFields.get(i).isNullable() || expectedFields.get(i).isNullable() + || Cast.castNullable(false, inputFields.get(i).getDataType(), newDataType.get()); + newFields.add(inputFields.get(i).withDataTypeAndNullable(newDataType.get(), nullable)); } else { return Optional.empty(); } @@ -458,7 +461,8 @@ private static boolean matchesType(DataType input, DataType target) { return false; } for (int i = 0; i < inputFields.size(); i++) { - if (!matchesType(inputFields.get(i).getDataType(), targetFields.get(i).getDataType())) { + if (inputFields.get(i).isNullable() != targetFields.get(i).isNullable() + || !matchesType(inputFields.get(i).getDataType(), targetFields.get(i).getDataType())) { return false; } } @@ -1216,7 +1220,11 @@ private static Optional findWiderComplexTypeForTwo( leftFields.get(i).getDataType(), rightFields.get(i).getDataType(), overflowToDouble, stringIsHighPriority); if (newDataType.isPresent()) { - newFields.add(leftFields.get(i).withDataType(newDataType.get())); + // The common layout must admit NULLs produced while either child is cast to it. + boolean nullable = leftFields.get(i).isNullable() || rightFields.get(i).isNullable() + || Cast.castNullable(false, leftFields.get(i).getDataType(), newDataType.get()) + || Cast.castNullable(false, rightFields.get(i).getDataType(), newDataType.get()); + newFields.add(leftFields.get(i).withDataTypeAndNullable(newDataType.get(), nullable)); } else { return Optional.empty(); } @@ -1736,7 +1744,11 @@ private static Optional findCommonComplexTypeForComparison( Optional newDataType = findWiderTypeForTwoForComparison(leftFields.get(i).getDataType(), rightFields.get(i).getDataType(), intStringToString); if (newDataType.isPresent()) { - newFields.add(leftFields.get(i).withDataType(newDataType.get())); + // The common layout must admit NULLs produced while either child is cast to it. + boolean nullable = leftFields.get(i).isNullable() || rightFields.get(i).isNullable() + || Cast.castNullable(false, leftFields.get(i).getDataType(), newDataType.get()) + || Cast.castNullable(false, rightFields.get(i).getDataType(), newDataType.get()); + newFields.add(leftFields.get(i).withDataTypeAndNullable(newDataType.get(), nullable)); } else { return Optional.empty(); } @@ -1981,7 +1993,12 @@ private static Optional findCommonComplexTypeForCaseWhen(DataType left Optional newDataType = findWiderTypeForTwoForCaseWhen(leftFields.get(i).getDataType(), rightFields.get(i).getDataType()); if (newDataType.isPresent()) { - newFields.add(leftFields.get(i).withDataType(newDataType.get())); + // Legacy CASE must admit both declared NULLs and NULLs introduced by either + // arm's cast, independent of the order in which arms are reduced. + boolean nullable = leftFields.get(i).isNullable() || rightFields.get(i).isNullable() + || Cast.castNullable(false, leftFields.get(i).getDataType(), newDataType.get()) + || Cast.castNullable(false, rightFields.get(i).getDataType(), newDataType.get()); + newFields.add(leftFields.get(i).withDataTypeAndNullable(newDataType.get(), nullable)); } else { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergDeleteSink.java index 19bc1af6f14252..484ddba6597d43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergDeleteSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergDeleteSink.java @@ -51,6 +51,7 @@ public class IcebergDeleteSink extends BaseExternalTableDataSink { private final IcebergExternalTable targetTable; + private final Table targetIcebergTable; private final DeleteCommandContext deleteContext; private List rewritableDeleteFileSets = Collections.emptyList(); @@ -62,19 +63,23 @@ public class IcebergDeleteSink extends BaseExternalTableDataSink { // Store PropertiesMap, including vended credentials or static credentials private Map storagePropertiesMap; - public IcebergDeleteSink(IcebergExternalTable targetTable, DeleteCommandContext deleteContext) { + public IcebergDeleteSink(IcebergExternalTable targetTable, Table targetIcebergTable, + DeleteCommandContext deleteContext) { super(); if (targetTable.isView()) { throw new UnsupportedOperationException("DELETE from iceberg view is not supported"); } this.targetTable = targetTable; + // Keep writer configuration on the generation scanned by DELETE so a concurrent commit cannot + // mix its spec, location, or credentials with delete files derived from an older generation. + this.targetIcebergTable = targetIcebergTable; this.deleteContext = deleteContext; IcebergExternalCatalog catalog = (IcebergExternalCatalog) targetTable.getCatalog(); storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( catalog.getCatalogProperty().getMetastoreProperties(), catalog.getCatalogProperty().getStoragePropertiesMap(), - targetTable.getIcebergTable()); + targetIcebergTable); } public void setRewritableDeleteFileSets(List deleteFileSets) { @@ -107,7 +112,7 @@ public void bindDataSink(Optional insertCtx) TIcebergDeleteSink tSink = new TIcebergDeleteSink(); - Table icebergTable = targetTable.getIcebergTable(); + Table icebergTable = targetIcebergTable; tSink.setDbName(targetTable.getDbName()); tSink.setTbName(targetTable.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java index c57ea76c75dd1f..5906e728f534f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java @@ -63,6 +63,7 @@ public class IcebergMergeSink extends BaseExternalTableDataSink { private final IcebergExternalTable targetTable; + private final Table targetIcebergTable; private final DeleteCommandContext deleteContext; private final boolean requireMergeCardinalityCheck; private List rewritableDeleteFileSets = Collections.emptyList(); @@ -77,11 +78,17 @@ public class IcebergMergeSink extends BaseExternalTableDataSink { public IcebergMergeSink(IcebergExternalTable targetTable, DeleteCommandContext deleteContext, boolean requireMergeCardinalityCheck) { + this(targetTable, targetTable.getIcebergTable(), deleteContext, requireMergeCardinalityCheck); + } + + public IcebergMergeSink(IcebergExternalTable targetTable, Table targetIcebergTable, + DeleteCommandContext deleteContext, boolean requireMergeCardinalityCheck) { super(); if (targetTable.isView()) { throw new UnsupportedOperationException("UPDATE on iceberg view is not supported"); } this.targetTable = targetTable; + this.targetIcebergTable = targetIcebergTable; this.deleteContext = deleteContext; this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; @@ -89,7 +96,7 @@ public IcebergMergeSink(IcebergExternalTable targetTable, DeleteCommandContext d storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( catalog.getCatalogProperty().getMetastoreProperties(), catalog.getCatalogProperty().getStoragePropertiesMap(), - targetTable.getIcebergTable()); + targetIcebergTable); } public void setRewritableDeleteFileSets(List deleteFileSets) { @@ -122,7 +129,8 @@ public void bindDataSink(Optional insertCtx) TIcebergMergeSink tSink = new TIcebergMergeSink(); - Table icebergTable = targetTable.getIcebergTable(); + // Serialize exactly the schema/spec that the analyzed merge plan and transaction retain. + Table icebergTable = targetIcebergTable; tSink.setDbName(targetTable.getDbName()); tSink.setTbName(targetTable.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java index b7d3da47cb4d45..4f643f61748140 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java @@ -53,6 +53,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -60,6 +61,7 @@ public class IcebergTableSink extends BaseExternalTableDataSink { private List outputExprs; private final IcebergExternalTable targetTable; + private final Table icebergTable; private static final HashSet supportedTypes = new HashSet() {{ add(TFileFormatType.FORMAT_ORC); add(TFileFormatType.FORMAT_PARQUET); @@ -75,11 +77,27 @@ public IcebergTableSink(IcebergExternalTable targetTable) { throw new UnsupportedOperationException("Write data to iceberg view is not supported"); } this.targetTable = targetTable; + this.icebergTable = targetTable.getIcebergTable(); IcebergExternalCatalog catalog = (IcebergExternalCatalog) targetTable.getCatalog(); storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( catalog.getCatalogProperty().getMetastoreProperties(), catalog.getCatalogProperty().getStoragePropertiesMap(), - targetTable.getIcebergTable()); + icebergTable); + } + + public IcebergTableSink(IcebergExternalTable targetTable, Table icebergTable) { + super(); + if (targetTable.isView()) { + throw new UnsupportedOperationException("Write data to iceberg view is not supported"); + } + this.targetTable = targetTable; + // Keep credentials and every writer option on the metadata generation pinned during analysis. + this.icebergTable = Objects.requireNonNull(icebergTable, "icebergTable is not null"); + IcebergExternalCatalog catalog = (IcebergExternalCatalog) targetTable.getCatalog(); + storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( + catalog.getCatalogProperty().getMetastoreProperties(), + catalog.getCatalogProperty().getStoragePropertiesMap(), + icebergTable); } @Override @@ -98,10 +116,9 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) { if (explainLevel == TExplainLevel.BRIEF) { return strBuilder.toString(); } - Table icebergTable = targetTable.getIcebergTable(); strBuilder.append(prefix).append("Table: ").append(icebergTable.name()).append("\n"); if (icebergTable.sortOrder().isSorted()) { - strBuilder.append(prefix).append(targetTable.getSortOrderSql()).append("\n"); + strBuilder.append(prefix).append(targetTable.getSortOrderSql(icebergTable)).append("\n"); } // TODO: explain partitions @@ -114,8 +131,6 @@ public void bindDataSink(Optional insertCtx) TIcebergTableSink tSink = new TIcebergTableSink(); - Table icebergTable = targetTable.getIcebergTable(); - tSink.setDbName(targetTable.getDbName()); tSink.setTbName(targetTable.getName()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java index ac8c8b65a7aeb6..f8017bb2a7f557 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java @@ -22,6 +22,8 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.InternalCatalog; @@ -54,4 +56,42 @@ public void testFetchResult() throws AnalysisException { Assert.assertEquals("The row size should be 6", 6, procResult.getRows().get(1).size()); } + + @Test + public void testCreateResultShowsNestedCommentsWhenCommentsRequested() { + StructType structType = new StructType( + new StructField("value", Type.INT, "nested-comment", true)); + Column column = new Column("info", structType, true, null, true, "", "top-level-comment"); + + ProcResult result = IndexSchemaProcNode.createResult( + Lists.newArrayList(column), null, + Lists.newArrayList(IndexSchemaProcNode.COMMENT_COLUMN_TITLE)); + + Assert.assertTrue(result.getRows().get(0).get(1).contains("nested-comment")); + Assert.assertEquals("top-level-comment", result.getRows().get(0).get(6)); + } + + @Test + public void testCreateResultPreservesNestedRequirednessWithAndWithoutComments() { + StructType structType = new StructType(Lists.newArrayList( + new StructField("required_value", Type.INT, "required-comment", false), + new StructField("optional_value", Type.INT, "optional-comment", true))); + Column column = new Column("info", structType, true, null, true, "", "top-level-comment"); + + String typeWithComments = IndexSchemaProcNode.createResult( + Lists.newArrayList(column), null, + Lists.newArrayList(IndexSchemaProcNode.COMMENT_COLUMN_TITLE)) + .getRows().get(0).get(1); + Assert.assertTrue(typeWithComments.contains( + "required_value:int not null comment 'required-comment'")); + Assert.assertTrue(typeWithComments.contains( + "optional_value:int comment 'optional-comment'")); + + String typeWithoutComments = IndexSchemaProcNode.createResult( + Lists.newArrayList(column), null, Lists.newArrayList()) + .getRows().get(0).get(1); + Assert.assertTrue(typeWithoutComments.contains("required_value:int not null")); + Assert.assertFalse(typeWithoutComments.contains("required-comment")); + Assert.assertFalse(typeWithoutComments.contains("optional-comment")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index 21a899ac673f00..f7a55ae2f6c3c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -21,15 +21,19 @@ import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TColumnCategory; import org.apache.doris.thrift.TExpr; +import org.apache.doris.thrift.TExprNodeType; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TFileScanSlotInfo; @@ -44,15 +48,19 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; public class FileQueryScanNodeTest { private static final long MB = 1024L * 1024L; private static final Method UPDATE_REQUIRED_SLOTS_METHOD; + private static final Method SET_COLUMN_POSITION_MAPPING_METHOD; static { try { UPDATE_REQUIRED_SLOTS_METHOD = FileQueryScanNode.class.getDeclaredMethod("updateRequiredSlots"); UPDATE_REQUIRED_SLOTS_METHOD.setAccessible(true); + SET_COLUMN_POSITION_MAPPING_METHOD = FileQueryScanNode.class.getDeclaredMethod("setColumnPositionMapping"); + SET_COLUMN_POSITION_MAPPING_METHOD.setAccessible(true); } catch (ReflectiveOperationException e) { throw new ExceptionInInitializerError(e); } @@ -162,4 +170,63 @@ public void testUpdateRequiredSlotsPreservesInlineDefaultValueExpr() throws Exce Assert.assertSame(defaultExpr, updatedSlotInfo.getDefaultValueExpr()); } + @Test + public void testColumnPositionMappingUsesRelationSnapshotSchema() throws Exception { + TestFileQueryScanNode node = new TestFileQueryScanNode(new SessionVariable()); + IcebergExternalTable externalTable = Mockito.mock(IcebergExternalTable.class); + MvccSnapshot relationSnapshot = Mockito.mock(MvccSnapshot.class); + Column oldColumn = new Column("old_name", Type.INT); + node.setTargetTable(externalTable); + node.getTupleDescriptor().setTable(externalTable); + node.setRelationSnapshot(Optional.of(relationSnapshot)); + Mockito.when(externalTable.getFullSchema(Optional.of(relationSnapshot))) + .thenReturn(Collections.singletonList(oldColumn)); + + SlotDescriptor slot = new SlotDescriptor(new SlotId(1), node.getTupleDescriptor()); + slot.setColumn(oldColumn); + node.getTupleDescriptor().addSlot(slot); + TFileScanSlotInfo slotInfo = new TFileScanSlotInfo(); + slotInfo.setSlotId(slot.getId().asInt()); + slotInfo.setCategory(TColumnCategory.REGULAR); + slotInfo.setIsFileSlot(true); + node.params = new TFileScanRangeParams(); + node.params.setRequiredSlots(Collections.singletonList(slotInfo)); + + SET_COLUMN_POSITION_MAPPING_METHOD.invoke(node); + + Assert.assertEquals(Collections.singletonList(0), node.params.getColumnIdxs()); + Mockito.verify(externalTable).getFullSchema(Optional.of(relationSnapshot)); + Mockito.verify(externalTable, Mockito.never()).loadSnapshot(Mockito.any(), Mockito.any()); + Mockito.verify(externalTable, Mockito.never()).getFullSchema(); + } + + @Test + public void testDefaultValueUsesRelationSnapshotSchema() throws Exception { + TestFileQueryScanNode node = new TestFileQueryScanNode(new SessionVariable()); + IcebergExternalTable externalTable = Mockito.mock(IcebergExternalTable.class); + MvccSnapshot relationSnapshot = Mockito.mock(MvccSnapshot.class); + Column historicalColumn = new Column("x", Type.INT, true); + Column latestSameNamedColumn = new Column( + "x", Type.INT, false, AggregateType.NONE, true, "42", ""); + node.setTargetTable(externalTable); + node.getTupleDescriptor().setTable(externalTable); + node.setRelationSnapshot(Optional.of(relationSnapshot)); + Mockito.when(externalTable.getBaseSchema(Optional.of(relationSnapshot), false)) + .thenReturn(Collections.singletonList(historicalColumn)); + Mockito.when(externalTable.getFullSchema(Optional.of(relationSnapshot))) + .thenReturn(Collections.singletonList(historicalColumn)); + Mockito.when(externalTable.getFullSchema()) + .thenReturn(Collections.singletonList(latestSameNamedColumn)); + + SlotDescriptor slot = new SlotDescriptor(new SlotId(1), node.getTupleDescriptor()); + slot.setColumn(historicalColumn); + node.getTupleDescriptor().addSlot(slot); + + node.initSchemaParams(); + + TExpr defaultExpr = node.params.getDefaultValueOfSrcSlot().get(slot.getId().asInt()); + Assert.assertEquals(TExprNodeType.NULL_LITERAL, defaultExpr.getNodes().get(0).getNodeType()); + Mockito.verify(externalTable, Mockito.never()).getFullSchema(); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiUtilsTest.java index 71ad096c7ce7c8..b02ca9a94437d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/HudiUtilsTest.java @@ -20,12 +20,16 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import com.google.common.collect.Maps; import mockit.Mock; @@ -33,16 +37,96 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.util.Option; import org.junit.Assert; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; public class HudiUtilsTest { + private MockedStatic envMockedStatic; + + @org.junit.After + public void tearDown() { + if (envMockedStatic != null) { + envMockedStatic.close(); + envMockedStatic = null; + } + } + + @Test + public void testResolveQueryInstantPrefersPinnedSnapshot() { + long pinnedInstant = 20260727123456789L; + MvccSnapshot snapshot = new HudiMvccSnapshot(new TablePartitionValues(), pinnedInstant); + HoodieTimeline timeline = Mockito.mock(HoodieTimeline.class); + HoodieInstant newerInstant = Mockito.mock(HoodieInstant.class); + Mockito.when(newerInstant.requestedTime()).thenReturn("20260727123500000"); + Mockito.when(timeline.lastInstant()).thenReturn(Option.of(newerInstant)); + + Assert.assertEquals(Long.toString(pinnedInstant), + HudiUtils.resolveQueryInstant(Optional.of(snapshot), Optional.empty(), timeline).orElse(null)); + Mockito.verify(timeline, Mockito.never()).lastInstant(); + } + + @Test + public void testLatestSnapshotUsesPartitionsFromCapturedInstant() { + long capturedInstant = 20260727123456789L; + HoodieTableMetaClient capturedClient = Mockito.mock(HoodieTableMetaClient.class); + HoodieTimeline capturedTimeline = Mockito.mock(HoodieTimeline.class); + HoodieInstant capturedHoodieInstant = Mockito.mock(HoodieInstant.class); + Mockito.when(capturedClient.getCommitsAndCompactionTimeline()).thenReturn(capturedTimeline); + Mockito.when(capturedTimeline.filterCompletedInstants()).thenReturn(capturedTimeline); + Mockito.when(capturedTimeline.lastInstant()).thenReturn(Option.of(capturedHoodieInstant)); + Mockito.when(capturedHoodieInstant.requestedTime()).thenReturn(Long.toString(capturedInstant)); + + HoodieTableMetaClient refreshedClient = Mockito.mock(HoodieTableMetaClient.class); + HoodieTimeline refreshedTimeline = Mockito.mock(HoodieTimeline.class); + HoodieInstant refreshedInstant = Mockito.mock(HoodieInstant.class); + Mockito.when(refreshedClient.getCommitsAndCompactionTimeline()).thenReturn(refreshedTimeline); + Mockito.when(refreshedTimeline.filterCompletedInstants()).thenReturn(refreshedTimeline); + Mockito.when(refreshedTimeline.lastInstant()).thenReturn(Option.of(refreshedInstant)); + Mockito.when(refreshedInstant.requestedTime()).thenReturn("20260727123500000"); + + TablePartitionValues capturedPartitions = new TablePartitionValues(); + TablePartitionValues refreshedPartitions = new TablePartitionValues(); + HudiExternalMetaCache hudiCache = Mockito.mock(HudiExternalMetaCache.class); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(100L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.useHiveSyncPartition()).thenReturn(false); + // The second client models cache replacement after the instant has already been captured. + Mockito.when(table.getHudiClient()).thenReturn(capturedClient, refreshedClient); + Mockito.when(hudiCache.getPartitionValues(table, false)).thenReturn(refreshedPartitions); + Mockito.when(hudiCache.getSnapshotPartitionValues( + table, Long.toString(capturedInstant), false)).thenReturn(capturedPartitions); + + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(cacheMgr.hudi(catalog.getId())).thenReturn(hudiCache); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + envMockedStatic = Mockito.mockStatic(Env.class); + envMockedStatic.when(Env::getCurrentEnv).thenReturn(env); + + HudiMvccSnapshot snapshot = HudiUtils.getHudiMvccSnapshot(Optional.empty(), table); + + Assert.assertEquals(capturedInstant, snapshot.getTimestamp()); + Assert.assertSame(capturedPartitions, snapshot.getTablePartitionValues()); + Mockito.verify(hudiCache).getSnapshotPartitionValues( + table, Long.toString(capturedInstant), false); + } + @Test public void testGetHudiSchemaWithCleanCommit() throws IOException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 556ff50225f8ea..af3be4475f71ac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -171,10 +171,6 @@ protected void runBeforeAll() throws Exception { }).when(spyTable).getFullSchema(); Mockito.doReturn(ImmutableList.of()).when(spyTable) .getPartitionColumns(ArgumentMatchers.any()); - IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( - IcebergPartitionInfo.empty(), new IcebergSnapshot(0L, 0L)); - Mockito.doReturn(new IcebergMvccSnapshot(snapshotCacheValue)).when(spyTable) - .loadSnapshot(ArgumentMatchers.any(), ArgumentMatchers.any()); Table mockedIcebergTable = Mockito.mock(Table.class); PartitionSpec mockedSpec = Mockito.mock(PartitionSpec.class); Mockito.doReturn(false).when(mockedSpec).isPartitioned(); @@ -187,6 +183,10 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedSpec).when(mockedIcebergTable).spec(); Mockito.doReturn(ImmutableMap.of()).when(mockedIcebergTable).specs(); Mockito.doReturn(icebergSchema).when(mockedIcebergTable).schema(); + IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(0L, 0L), Optional.empty(), mockedIcebergTable); + Mockito.doReturn(new IcebergMvccSnapshot(snapshotCacheValue)).when(spyTable) + .loadSnapshot(ArgumentMatchers.any(), ArgumentMatchers.any()); // The scan now resolves initial defaults from the statement-pinned schema id, so the // mocked table must expose the same historical-schema lookup as a real Iceberg table. Mockito.doAnswer(invocation -> ImmutableMap.of( @@ -674,10 +674,8 @@ public void testIcebergUpdateExchangeUsesMergePartitioningWhenEnabled() throws E @Test public void testIcebergUpdateExchangeUsesPartitionColumnsWhenEnabled() throws Exception { useIceberg(); - IcebergExternalTable table = getIcebergTable(); - Column partitionColumn = new Column("age", PrimitiveType.INT); - Mockito.doReturn(ImmutableList.of(partitionColumn)).when(table) - .getPartitionColumns(ArgumentMatchers.any()); + PartitionSpec partitionSpec = PartitionSpec.builderFor(baseIcebergSchema).identity("age").build(); + Mockito.doReturn(partitionSpec).when(mockedIcebergTable).spec(); boolean previous = connectContext.getSessionVariable().enableIcebergMergePartitioning; connectContext.getSessionVariable().enableIcebergMergePartitioning = true; try { @@ -707,17 +705,15 @@ public void testIcebergUpdateExchangeUsesPartitionColumnsWhenEnabled() throws Ex Assertions.assertEquals(rowIdExprId, spec.getDeletePartitionExprIds().get(0)); } finally { connectContext.getSessionVariable().enableIcebergMergePartitioning = previous; - Mockito.doReturn(ImmutableList.of()).when(table).getPartitionColumns(ArgumentMatchers.any()); + Mockito.doReturn(basePartitionSpec).when(mockedIcebergTable).spec(); } } @Test public void testIcebergUpdatePartitionExpressionUsesPartitionColumnWhenEnabled() throws Exception { useIceberg(); - IcebergExternalTable table = getIcebergTable(); - Column partitionColumn = new Column("age", PrimitiveType.INT); - Mockito.doReturn(ImmutableList.of(partitionColumn)).when(table) - .getPartitionColumns(ArgumentMatchers.any()); + PartitionSpec partitionSpec = PartitionSpec.builderFor(baseIcebergSchema).identity("age").build(); + Mockito.doReturn(partitionSpec).when(mockedIcebergTable).spec(); boolean previous = connectContext.getSessionVariable().enableIcebergMergePartitioning; connectContext.getSessionVariable().enableIcebergMergePartitioning = true; try { @@ -746,7 +742,7 @@ public void testIcebergUpdatePartitionExpressionUsesPartitionColumnWhenEnabled() Assertions.assertEquals(ImmutableList.of(rowIdExprId), spec.getDeletePartitionExprIds()); } finally { connectContext.getSessionVariable().enableIcebergMergePartitioning = previous; - Mockito.doReturn(ImmutableList.of()).when(table).getPartitionColumns(ArgumentMatchers.any()); + Mockito.doReturn(basePartitionSpec).when(mockedIcebergTable).spec(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java new file mode 100644 index 00000000000000..03d217c25d16e5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.iceberg.MetadataTableType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class IcebergSysExternalTableTest { + @Test + public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(Mockito.mock(IcebergExternalCatalog.class)); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + + for (MetadataTableType type : new MetadataTableType[] { + MetadataTableType.HISTORY, + MetadataTableType.SNAPSHOTS, + MetadataTableType.REFS, + MetadataTableType.METADATA_LOG_ENTRIES}) { + IcebergSysExternalTable table = new IcebergSysExternalTable(sourceTable, type.name()); + Assertions.assertFalse(table.supportsSnapshotSelection(), type.name()); + } + + IcebergSysExternalTable dataFiles = new IcebergSysExternalTable( + sourceTable, MetadataTableType.DATA_FILES.name()); + Assertions.assertTrue(dataFiles.supportsSnapshotSelection()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index f3dd3166afb19c..e2d923f3438863 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; @@ -39,6 +40,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.UnboundPredicate; @@ -208,7 +210,7 @@ public void testPartitionedTable() throws UserException { .thenCallRealMethod(); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(ctdList); - txn.beginInsert(icebergExternalTable, Optional.empty()); + txn.beginInsert(icebergExternalTable, table, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); } @@ -321,7 +323,7 @@ public void testUnPartitionedTable() throws UserException { IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(ctdList); - txn.beginInsert(icebergExternalTable, Optional.empty()); + txn.beginInsert(icebergExternalTable, table, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); } @@ -433,7 +435,7 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(ctdList); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); - txn.beginInsert(icebergExternalTable, Optional.of(ctx)); + txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); ctx.setOverwrite(true); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); @@ -458,7 +460,7 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { IcebergTransaction txn = getTxn(); IcebergInsertCommandContext ctx = new IcebergInsertCommandContext(); - txn.beginInsert(icebergExternalTable, Optional.of(ctx)); + txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); ctx.setOverwrite(true); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); @@ -506,7 +508,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(ctdList); - txn.beginInsert(icebergExternalTable, Optional.empty()); + txn.beginInsert(icebergExternalTable, table, Optional.empty()); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); } @@ -528,7 +530,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th staticPartitions.put("dt4", "2024-12-11"); staticPartitions.put("str1", "partition-a"); ctx.setStaticPartitionValues(staticPartitions); - txn.beginInsert(icebergExternalTable, Optional.of(ctx)); + txn.beginInsert(icebergExternalTable, table, Optional.of(ctx)); txn.finishInsert(NameMapping.createForTest(dbName, tbWithPartition)); txn.commit(); } @@ -598,7 +600,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use ArgumentMatchers.anyList())) .thenReturn(Collections.singletonList(newDeleteFile)); - txn.beginDelete(icebergExternalTable); + txn.beginDelete(icebergExternalTable, icebergTable); txn.setRewrittenDeleteFilesByReferencedDataFile( Collections.singletonMap(referencedDataFile, Arrays.asList(oldDeleteFile1, oldDeleteFile2))); txn.finishDelete(NameMapping.createForTest(dbName, tbWithoutPartition)); @@ -659,7 +661,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect ArgumentMatchers.anyList())) .thenReturn(Collections.singletonList(newDeleteFile)); - txn.beginDelete(icebergExternalTable); + txn.beginDelete(icebergExternalTable, icebergTable); txn.setRewrittenDeleteFilesByReferencedDataFile( Collections.singletonMap(referencedDataFile, Collections.singletonList(oldDeleteFile))); txn.finishDelete(NameMapping.createForTest(dbName, tbWithoutPartition)); @@ -674,6 +676,147 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect Mockito.verify(rowDelta).commit(); } + @Test + public void testBeginInsertUsesRetainedTargetTable() throws UserException { + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn("retained_target"); + Table retainedTable = Mockito.mock(Table.class); + org.apache.iceberg.Transaction retainedTransaction = + Mockito.mock(org.apache.iceberg.Transaction.class); + Mockito.when(retainedTable.newTransaction()).thenReturn(retainedTransaction); + + IcebergTransaction txn = getTxn(); + txn.beginInsert(dorisTable, retainedTable, Optional.empty()); + + Mockito.verify(retainedTable).newTransaction(); + } + + @Test + public void testBeginDeleteUsesRetainedTargetTable() throws UserException { + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn("retained_delete_target"); + Table retainedTable = Mockito.mock(Table.class); + org.apache.iceberg.Transaction retainedTransaction = + Mockito.mock(org.apache.iceberg.Transaction.class); + Mockito.when(retainedTable.newTransaction()).thenReturn(retainedTransaction); + + IcebergTransaction txn = getTxn(); + txn.beginDelete(dorisTable, retainedTable); + + Mockito.verify(retainedTable).newTransaction(); + } + + @Test + public void testRetainedGenerationCommitsThroughWritableOperations() throws UserException { + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), liveTable); + Table retainedTable = cacheValue.getIcebergTable().get(); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("retained-generation.parquet"); + commitData.setFileContent(TFileContent.DATA); + commitData.setRowCount(1); + commitData.setFileSize(1); + + try (MockedStatic mockedUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + IcebergTransaction txn = getTxn(); + txn.updateIcebergCommitData(Collections.singletonList(commitData)); + txn.beginInsert(dorisTable, retainedTable, Optional.empty()); + txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); + txn.commit(); + } + + Assert.assertNotNull(ops.getCatalog().loadTable( + TableIdentifier.of(dbName, tbWithoutPartition)).currentSnapshot()); + } + + @Test + public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws UserException { + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), liveTable); + Table retainedTable = cacheValue.getIcebergTable().get(); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("stale-retained-generation.parquet"); + commitData.setFileContent(TFileContent.DATA); + commitData.setRowCount(1); + commitData.setFileSize(1); + + try (MockedStatic mockedUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + IcebergTransaction txn = getTxn(); + txn.updateIcebergCommitData(Collections.singletonList(commitData)); + txn.beginInsert(dorisTable, retainedTable, Optional.empty()); + txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); + + liveTable.updateProperties().set("concurrent-update", "true").commit(); + Assert.assertThrows(CommitFailedException.class, txn::commit); + } + } + + @Test + public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws UserException, IOException { + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), liveTable); + Table retainedTable = cacheValue.getIcebergTable().get(); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("retry-after-concurrent-data-commit.parquet"); + commitData.setFileContent(TFileContent.DATA); + commitData.setRowCount(1); + commitData.setFileSize(1); + + try (MockedStatic mockedUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + IcebergTransaction txn = getTxn(); + txn.updateIcebergCommitData(Collections.singletonList(commitData)); + txn.beginInsert(dorisTable, retainedTable, Optional.empty()); + txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); + + Path concurrentFile = Files.createTempFile("concurrent-data-commit-", ".parquet"); + liveTable.newFastAppend() + .appendFile(DataFiles.builder(liveTable.spec()) + .withPath(concurrentFile.toString()) + .withFileSizeInBytes(1) + .withRecordCount(1) + .withFormat(FileFormat.PARQUET) + .build()) + .commit(); + txn.commit(); + } + + Table refreshedTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + Assert.assertEquals(2, refreshedTable.history().size()); + } + + @Test + public void testStaticPartitionFilterRejectsUnknownKey() { + Schema schema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "day", Types.StringType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("day").build(); + + Assert.assertThrows(IllegalArgumentException.class, + () -> getTxn().buildPartitionFilter( + Collections.singletonMap("unknown", "2026-01-01"), spec, schema)); + } + private DeleteFile buildDeletionVectorDeleteFile(String puffinPath, String referencedDataFile, long contentOffset, long contentLength) { return FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index c9b5f13080c319..f692e83031490a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -22,9 +22,11 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import com.google.common.collect.ImmutableMap; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.GenericPartitionFieldSummary; import org.apache.iceberg.HistoryEntry; import org.apache.iceberg.ManifestContent; @@ -37,6 +39,8 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableProperties; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -67,8 +71,70 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; public class IcebergUtilsTest { + @Test + public void testSnapshotCacheFreezesSharedTableOperations() { + Schema originalSchema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get())); + Schema evolvedSchema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "payload", Types.StringType.get())); + AtomicReference currentMetadata = new AtomicReference<>( + TableMetadata.newTableMetadata(originalSchema, PartitionSpec.unpartitioned(), + "file:/tmp/iceberg-cache-table", Collections.emptyMap())); + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenAnswer(invocation -> currentMetadata.get()); + Mockito.when(operations.io()).thenReturn(Mockito.mock(org.apache.iceberg.io.FileIO.class)); + Mockito.when(operations.locationProvider()) + .thenReturn(Mockito.mock(org.apache.iceberg.io.LocationProvider.class)); + Table sharedTable = new BaseTable(operations, "table"); + + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, originalSchema.schemaId()), + Optional.empty(), sharedTable); + currentMetadata.set(TableMetadata.newTableMetadata(evolvedSchema, PartitionSpec.unpartitioned(), + "file:/tmp/iceberg-cache-table", Collections.emptyMap())); + + Assert.assertEquals(2, sharedTable.schema().columns().size()); + Assert.assertEquals(1, cacheValue.getIcebergTable().get().schema().columns().size()); + } + + @Test + public void testRetainedGenerationKeepsProjectionAtomic() { + Schema originalSchema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get())); + Schema evolvedSchema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "payload", Types.StringType.get())); + TableMetadata originalMetadata = TableMetadata.newTableMetadata( + originalSchema, PartitionSpec.unpartitioned(), + "file:/tmp/iceberg-atomic-projection", Collections.emptyMap()); + TableMetadata evolvedMetadata = TableMetadata.newTableMetadata( + evolvedSchema, PartitionSpec.unpartitioned(), + "file:/tmp/iceberg-atomic-projection", + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, + "{\"type\":\"struct\",\"fields\":[{\"field-id\":2,\"names\":[\"payload\"]}]}")); + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(originalMetadata, evolvedMetadata); + Mockito.when(operations.io()).thenReturn(Mockito.mock(org.apache.iceberg.io.FileIO.class)); + Mockito.when(operations.locationProvider()) + .thenReturn(Mockito.mock(org.apache.iceberg.io.LocationProvider.class)); + Table sharedTable = new BaseTable(operations, "table"); + + Table retainedTable = IcebergSnapshotCacheValue.retainTableGeneration(sharedTable); + IcebergSnapshot snapshot = IcebergUtils.getLatestIcebergSnapshot(retainedTable); + Optional>> nameMapping = IcebergUtils.getNameMapping(retainedTable); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), snapshot, nameMapping, retainedTable); + + Assert.assertEquals(originalSchema.schemaId(), snapshot.getSchemaId()); + Assert.assertFalse(nameMapping.isPresent()); + Assert.assertEquals(1, cacheValue.getIcebergTable().get().schema().columns().size()); + Mockito.verify(operations, Mockito.times(1)).current(); + } + @Test public void testGetFileFormatUsesPropertiesWithoutPlanningDataFiles() { Table table = Mockito.mock(Table.class); @@ -91,6 +157,34 @@ public void testGetFileFormatUsesConfiguredTableFormat() { Mockito.verify(table, Mockito.never()).newScan(); } + @Test + public void testPartitionColumnsUseFrozenTableSpec() { + Schema frozenSchema = new Schema(17, Arrays.asList( + Types.NestedField.required(1, "p", Types.IntegerType.get()), + Types.NestedField.optional(2, "q", Types.IntegerType.get()))); + Table frozenTable = Mockito.mock(Table.class); + Mockito.when(frozenTable.schema()).thenReturn(frozenSchema); + Mockito.when(frozenTable.schemas()).thenReturn( + Collections.singletonMap(frozenSchema.schemaId(), frozenSchema)); + Mockito.when(frozenTable.spec()).thenReturn(PartitionSpec.builderFor(frozenSchema).identity("p").build()); + Mockito.when(frozenTable.currentSnapshot()).thenReturn(Mockito.mock(Snapshot.class)); + + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(dorisTable.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(101L, frozenSchema.schemaId()), + Optional.empty(), frozenTable); + + List partitionColumns = IcebergUtils.getIcebergPartitionColumns( + Optional.of(new IcebergMvccSnapshot(cacheValue)), dorisTable); + + Assert.assertEquals(Collections.singletonList("p"), partitionColumns.stream() + .map(Column::getName).collect(java.util.stream.Collectors.toList())); + } + @Test public void testParseTableName() { try { @@ -239,6 +333,19 @@ public void testParseSchemaPreservesNonLowercaseColumnNames() { Assert.assertEquals("PART", columns.get(1).getName()); } + @Test + public void testParseSchemaPreservesTopLevelAndNestedComments() { + Schema schema = new Schema(Types.NestedField.optional( + 1, "info", Types.StructType.of( + Types.NestedField.optional(2, "value", Types.IntegerType.get(), "nested-comment")), + "top-level-comment")); + + List columns = IcebergUtils.parseSchema(schema, false, false); + + Assert.assertEquals("top-level-comment", columns.get(0).getComment()); + Assert.assertTrue(columns.get(0).getType().toSql().contains("comment 'nested-comment'")); + } + @Test public void testParseSchemaPreservesInitialDefault() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java new file mode 100644 index 00000000000000..e297ba478e46bf --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/rewrite/RewriteDataFileExecutorTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg.rewrite; + +import org.apache.doris.catalog.Env; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergTransaction; + +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +class RewriteDataFileExecutorTest { + + @Test + void testInvalidateTableCacheAfterCommit() throws Exception { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + IcebergTransaction transaction = Mockito.mock(IcebergTransaction.class); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + new RewriteDataFileExecutor(table, null).commitAndInvalidate(transaction); + + InOrder inOrder = Mockito.inOrder(transaction, cacheMgr); + inOrder.verify(transaction).commit(); + inOrder.verify(cacheMgr).invalidateTableCache(table); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index bab7e8085fbbe3..b72a7d500701da 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -17,8 +17,12 @@ package org.apache.doris.datasource.iceberg.source; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; @@ -31,12 +35,15 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; import org.apache.doris.datasource.iceberg.IcebergSnapshot; import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccTableInfo; import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; @@ -52,16 +59,22 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.iceberg.BaseMetadataTable; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.types.Types; @@ -92,6 +105,12 @@ private static Optional>> extractNameMapping( return (Optional>>) method.invoke(node); } + private static Table useFrozenTableGeneration(IcebergScanNode node, Table table) throws Exception { + Method method = IcebergScanNode.class.getDeclaredMethod("useFrozenTableGeneration", Table.class); + method.setAccessible(true); + return (Table) method.invoke(node, table); + } + private static class TestIcebergScanNode extends IcebergScanNode { private final boolean enableMappingVarbinary; private TableScan tableScan; @@ -114,6 +133,10 @@ public TableScan createTableScan() { return tableScan; } + TableScan createRealTableScan() throws UserException { + return super.createTableScan(); + } + @Override public boolean isBatchMode() { return false; @@ -174,7 +197,8 @@ public void testPartitionEvolutionKeepsNonFileSlotInReaderSchema() throws Except projectedColumn.setUniqueId(2); IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); - Mockito.when(targetTable.getColumns()).thenReturn( + // Reader schema resolution is pinned to the relation snapshot, including partition-only columns. + Mockito.when(targetTable.getFullSchema(Mockito.>any())).thenReturn( ImmutableList.of(evolvedIdentityColumn, projectedColumn)); IcebergSource source = Mockito.mock(IcebergSource.class); Mockito.when(source.getTargetTable()).thenReturn(targetTable); @@ -509,6 +533,297 @@ public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() th Mockito.verify(node, Mockito.never()).createTableScan(); } + @Test + public void testHistoricalPredicateUsesSelectedScanSchema() throws Exception { + Schema historicalSchema = new Schema( + Types.NestedField.optional(7, "old_name", Types.IntegerType.get())); + Schema currentSchema = new Schema( + Types.NestedField.optional(8, "new_name", Types.IntegerType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(currentSchema); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap( + historicalSchema.schemaId(), historicalSchema)); + TableScan scan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(scan.schema()).thenReturn(historicalSchema); + Mockito.when(scan.metricsReporter(Mockito.any())).thenReturn(scan); + Mockito.when(scan.useSnapshot(1L)).thenReturn(scan); + Mockito.when(scan.project(historicalSchema)).thenReturn(scan); + Mockito.when(scan.filter(Mockito.any())).thenReturn(scan); + Mockito.when(scan.planWith(Mockito.any())).thenReturn(scan); + Mockito.when(table.newScan()).thenReturn(scan); + + IcebergSource source = Mockito.mock(IcebergSource.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(source.getCatalog()).thenReturn(catalog); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(new IcebergTableQueryInfo(1L, null, historicalSchema.schemaId())) + .when(node).getSpecifiedSnapshot(); + node.addConjunct(new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(new TableName(), "old_name"), new IntLiteral(1, Type.INT))); + + node.createRealTableScan(); + + Mockito.verify(scan).filter(Mockito.argThat(expression -> expression.toString().contains("old_name"))); + } + + @Test + public void testPinnedBranchUsesFrozenSnapshotWithCurrentSchema() throws Exception { + Schema snapshotSchema = new Schema(11, ImmutableList.of( + Types.NestedField.optional(1, "old_name", Types.StringType.get()))); + Schema currentSchema = new Schema(12, ImmutableList.of( + Types.NestedField.optional(1, "new_name", Types.StringType.get()), + Types.NestedField.optional(2, "new_col", Types.StringType.get()))); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(currentSchema); + Mockito.when(table.schemas()).thenReturn(ImmutableMap.of(11, snapshotSchema, 12, currentSchema)); + Mockito.when(table.refs()).thenReturn(Collections.singletonMap( + "moving", SnapshotRef.branchBuilder(2L).build())); + TableScan scan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(scan.schema()).thenReturn(currentSchema); + Mockito.when(scan.metricsReporter(Mockito.any())).thenReturn(scan); + Mockito.when(scan.useSnapshot(1L)).thenReturn(scan); + Mockito.when(scan.project(currentSchema)).thenReturn(scan); + Mockito.when(table.newScan()).thenReturn(scan); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergExternalCatalog sourceCatalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + Mockito.when(source.getCatalog()).thenReturn(sourceCatalog); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, table); + setIcebergSource(node, source); + node.setScanParams(new TableScanParams(TableScanParams.BRANCH, + Collections.singletonMap(TableScanParams.PARAMS_NAME, "moving"), Collections.emptyList())); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, currentSchema.schemaId())))); + try { + node.createRealTableScan(); + + Mockito.verify(scan).useSnapshot(1L); + Mockito.verify(scan).project(currentSchema); + Mockito.verify(scan, Mockito.never()).useRef(Mockito.anyString()); + Mockito.verify(table, Mockito.never()).refs(); + } finally { + statementContext.close(); + ConnectContext.remove(); + } + } + + @Test + public void testPinnedLatestUsesFrozenSnapshot() throws Exception { + Schema schema = new Schema(21, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.IntegerType.get()))); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(schema.schemaId(), schema)); + TableScan scan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(scan.schema()).thenReturn(schema); + Mockito.when(scan.metricsReporter(Mockito.any())).thenReturn(scan); + Mockito.when(scan.useSnapshot(7L)).thenReturn(scan); + Mockito.when(scan.project(schema)).thenReturn(scan); + Mockito.when(table.newScan()).thenReturn(scan); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + Mockito.when(source.getCatalog()).thenReturn(Mockito.mock(IcebergExternalCatalog.class)); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, table); + setIcebergSource(node, source); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(7L, schema.schemaId()))))); + + node.createRealTableScan(); + + Mockito.verify(scan).useSnapshot(7L); + Mockito.verify(scan).project(schema); + } + + @Test + public void testPinnedEmptyTableDoesNotUseInvalidSnapshot() throws Exception { + Schema schema = new Schema(21, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.IntegerType.get()))); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(schema.schemaId(), schema)); + TableScan scan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(scan.schema()).thenReturn(schema); + Mockito.when(scan.metricsReporter(Mockito.any())).thenReturn(scan); + Mockito.when(table.newScan()).thenReturn(scan); + + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + Mockito.when(source.getCatalog()).thenReturn(Mockito.mock(IcebergExternalCatalog.class)); + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, table); + setIcebergSource(node, source); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(-1L, schema.schemaId()))))); + + node.createRealTableScan(); + + Mockito.verify(scan, Mockito.never()).useSnapshot(Mockito.anyLong()); + } + + @Test + public void testPinnedEmptyTableUsesFrozenGenerationAfterRefresh() throws Exception { + Schema schema = new Schema(21, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.IntegerType.get()))); + Table frozenEmptyTable = Mockito.mock(Table.class); + TableScan frozenEmptyScan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(frozenEmptyScan.schema()).thenReturn(schema); + Mockito.when(frozenEmptyScan.metricsReporter(Mockito.any())).thenReturn(frozenEmptyScan); + Mockito.when(frozenEmptyScan.planWith(Mockito.any())).thenReturn(frozenEmptyScan); + Mockito.when(frozenEmptyTable.newScan()).thenReturn(frozenEmptyScan); + + Table refreshedTable = Mockito.mock(Table.class); + TableScan refreshedScan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(refreshedScan.schema()).thenReturn(schema); + Mockito.when(refreshedScan.metricsReporter(Mockito.any())).thenReturn(refreshedScan); + Mockito.when(refreshedScan.planWith(Mockito.any())).thenReturn(refreshedScan); + Mockito.when(refreshedTable.newScan()).thenReturn(refreshedScan); + + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + Mockito.when(source.getCatalog()).thenReturn(Mockito.mock(IcebergExternalCatalog.class)); + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(-1L, schema.schemaId()), Optional.empty(), frozenEmptyTable)))); + + TableScan plannedScan = node.createRealTableScan(); + + Assert.assertSame(frozenEmptyScan, plannedScan); + Mockito.verify(frozenEmptyTable).newScan(); + Mockito.verify(refreshedTable, Mockito.never()).newScan(); + } + + @Test + public void testPinnedNonEmptyTableUsesFrozenGenerationAfterRefresh() throws Exception { + Schema schema = new Schema(21, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.IntegerType.get()))); + Table frozenTable = Mockito.mock(Table.class); + TableScan frozenScan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(frozenScan.schema()).thenReturn(schema); + Mockito.when(frozenScan.metricsReporter(Mockito.any())).thenReturn(frozenScan); + Mockito.when(frozenScan.useSnapshot(101L)).thenReturn(frozenScan); + Mockito.when(frozenScan.project(schema)).thenReturn(frozenScan); + Mockito.when(frozenScan.planWith(Mockito.any())).thenReturn(frozenScan); + Mockito.when(frozenTable.newScan()).thenReturn(frozenScan); + Mockito.when(frozenTable.schemas()).thenReturn(Collections.singletonMap(schema.schemaId(), schema)); + + Table refreshedTable = Mockito.mock(Table.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + Mockito.when(source.getCatalog()).thenReturn(Mockito.mock(IcebergExternalCatalog.class)); + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(101L, schema.schemaId()), Optional.empty(), frozenTable)))); + + TableScan plannedScan = node.createRealTableScan(); + + Assert.assertSame(frozenScan, plannedScan); + Mockito.verify(frozenScan).useSnapshot(101L); + Mockito.verify(refreshedTable, Mockito.never()).newScan(); + } + + @Test + public void testSnapshotSelectableMetadataTableUsesFrozenBaseGeneration() throws Exception { + Schema schema = new Schema(21, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.IntegerType.get()))); + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:/tmp/frozen-metadata-table", + Collections.emptyMap()); + Table frozenBaseTable = new BaseTable(new StaticTableOperations( + metadata, Mockito.mock(org.apache.iceberg.io.FileIO.class), + Mockito.mock(org.apache.iceberg.io.LocationProvider.class)), "table"); + Table currentMetadataTable = Mockito.mock(Table.class); + + IcebergSysExternalTable targetTable = Mockito.mock(IcebergSysExternalTable.class); + Mockito.when(targetTable.supportsSnapshotSelection()).thenReturn(true); + Mockito.when(targetTable.getSysTableType()).thenReturn(MetadataTableType.FILES.name()); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergSource(node, source); + Field isSystemTableField = IcebergScanNode.class.getDeclaredField("isSystemTable"); + isSystemTableField.setAccessible(true); + isSystemTableField.setBoolean(node, true); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(-1L, schema.schemaId()), Optional.empty(), frozenBaseTable)))); + + Table retainedMetadataTable = useFrozenTableGeneration(node, currentMetadataTable); + + Assert.assertNotSame(currentMetadataTable, retainedMetadataTable); + Assert.assertTrue(retainedMetadataTable instanceof BaseMetadataTable); + Assert.assertEquals(schema.asStruct(), ((BaseMetadataTable) retainedMetadataTable).table() + .schema().asStruct()); + } + + @Test + public void testAllMetadataTableDoesNotUseSnapshot() throws Exception { + Schema schema = new Schema(21, ImmutableList.of( + Types.NestedField.optional(1, "id", Types.IntegerType.get()))); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(schema.schemaId(), schema)); + TableScan scan = Mockito.mock(TableScan.class, Mockito.RETURNS_SELF); + Mockito.when(scan.schema()).thenReturn(schema); + Mockito.when(scan.metricsReporter(Mockito.any())).thenReturn(scan); + Mockito.when(table.newScan()).thenReturn(scan); + + IcebergSysExternalTable targetTable = Mockito.mock(IcebergSysExternalTable.class); + Mockito.when(targetTable.supportsSnapshotSelection()).thenReturn(false); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + Mockito.when(source.getCatalog()).thenReturn(Mockito.mock(IcebergExternalCatalog.class)); + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, table); + setIcebergSource(node, source); + node.setRelationSnapshot(Optional.of(new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(7L, schema.schemaId()))))); + + node.createRealTableScan(); + + Mockito.verify(scan, Mockito.never()).useSnapshot(Mockito.anyLong()); + } + private static void setIcebergTable(IcebergScanNode node, Table table) throws Exception { Field icebergTableField = IcebergScanNode.class.getDeclaredField("icebergTable"); icebergTableField.setAccessible(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java index 6b0116433514f9..a6978b511f990a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java @@ -18,11 +18,16 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.catalog.Column; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import com.google.common.collect.ImmutableMap; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentMatchers; @@ -30,6 +35,7 @@ import org.mockito.Mockito; import java.util.Collections; +import java.util.List; import java.util.Optional; public class PaimonExternalTableTest { @@ -94,4 +100,27 @@ public void testModeOnlyLatestUsesStatementSnapshotAcrossPhases() { && options.containsKey("scan.mode") && options.get("scan.mode") == null)); } + + @Test + public void testBranchSnapshotUsesEffectiveTableSchema() { + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = new PaimonExternalTable( + 1L, "local_table", "remote_table", catalog, database); + FileStoreTable branchTable = Mockito.mock(FileStoreTable.class); + TableSchema branchSchema = new TableSchema(3L, + Collections.singletonList(new DataField(1, "branch_column", DataTypes.INT())), + 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), ""); + Mockito.when(branchTable.schema()).thenReturn(branchSchema); + Mockito.when(branchTable.schemaManager()).thenThrow( + new AssertionError("branch schema must not be looked up through the base namespace")); + PaimonSnapshotCacheValue cacheValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(7L, 3L, branchTable), true); + + List schema = externalTable.getFullSchema( + Optional.of(new PaimonMvccSnapshot(cacheValue))); + + Assert.assertEquals(1, schema.size()); + Assert.assertEquals("branch_column", schema.get(0).getName()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 6b2d32f085f1b2..5a3a1788de6c41 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -27,12 +27,18 @@ import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.FileSplitter; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonMvccSnapshot; +import org.apache.doris.datasource.paimon.PaimonPartitionInfo; import org.apache.doris.datasource.paimon.PaimonScanParams; +import org.apache.doris.datasource.paimon.PaimonSnapshot; +import org.apache.doris.datasource.paimon.PaimonSnapshotCacheValue; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; import org.apache.doris.datasource.paimon.PaimonUtil; +import org.apache.doris.datasource.paimon.PaimonUtils; import org.apache.doris.datasource.property.metastore.MetastoreProperties; import org.apache.doris.datasource.property.metastore.PaimonJdbcMetaStoreProperties; import org.apache.doris.planner.PlanNodeId; @@ -56,6 +62,7 @@ import org.apache.paimon.table.AppendOnlyFileStoreTable; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.DataSplit; @@ -71,6 +78,7 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentMatchers; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @@ -687,6 +695,9 @@ public void testPinnedFileCreationScanPreservesBatchReaderFilters() throws Excep Collections.emptyList()); scanParams.getOrResolveMapParams(options -> PaimonScanParams.resolveOptions(table, options)); node.setScanParams(scanParams); + // This reader-filter test deliberately has no bound MVCC snapshot; avoid asking the + // otherwise unstubbed external-table mock to manufacture one from scan options. + node.setRelationSnapshot(Optional.empty()); Assert.assertTrue(node.getPaimonSplitFromAPI().isEmpty()); @@ -695,6 +706,20 @@ public void testPinnedFileCreationScanPreservesBatchReaderFilters() throws Excep Mockito.verify(reader).onlyReadRealBuckets(); } + @Test + public void testBoundEmptySnapshotDoesNotReadLaterCommit() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Table liveTable = Mockito.mock(Table.class); + node.setSource(source); + node.setRelationSnapshot(Optional.of(new PaimonMvccSnapshot( + new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(PaimonSnapshot.INVALID_SNAPSHOT_ID, 1L, liveTable))))); + + Assert.assertTrue(node.getPaimonSplitFromAPI().isEmpty()); + Mockito.verify(liveTable, Mockito.never()).newReadBuilder(); + } + @Test public void testSystemTableRejectsIncrementalReadWhenReaderIgnoresRange() throws Exception { PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); @@ -837,7 +862,9 @@ public void testLatestScanUsesRefreshedDescriptorColumnPositions() throws Except PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); Column latestColumn = Mockito.mock(Column.class); Mockito.when(latestColumn.getName()).thenReturn("renamed_name"); - Mockito.when(node.getTupleDesc().getTable().getFullSchema()) + PaimonExternalTable externalTable = (PaimonExternalTable) node.getTupleDesc().getTable(); + // File scan metadata is resolved against the relation snapshot, so mock the snapshot-aware lookup. + Mockito.when(externalTable.getFullSchema(Mockito.>any())) .thenReturn(Collections.singletonList(latestColumn)); Table staleTableHandle = Mockito.mock(Table.class); @@ -1168,8 +1195,12 @@ public void testNativeSplitCarriesPartitionMetadataWithoutRuntimeFilterPruning() } @Test - public void testSetPaimonParamsUsesJniForDataSplit() throws Exception { - PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + public void testSetPaimonParamsUsesJniWhenCppOptionEnabled() throws Exception { + // Keep this as real session state because the JNI-only path need not read the option; + // strict mocks should not make the test depend on whether that implementation detail is consulted. + SessionVariable cppEnabledSession = new SessionVariable(); + cppEnabledSession.setEnablePaimonCppReader(true); + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), cppEnabledSession); PaimonSource source = Mockito.mock(PaimonSource.class); Table paimonTable = mockPaimonTableWithPartitionKeys(Collections.emptyList()); Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); @@ -1194,6 +1225,33 @@ public void testGetFieldIndexMatchesMixedCaseColumns() { Assert.assertEquals(-1, PaimonScanNode.getFieldIndex(fieldNames, "missing_col")); } + @Test + public void testHistorySchemaUsesRelationPaimonTable() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + DataTable branchTable = Mockito.mock(DataTable.class, Mockito.RETURNS_DEEP_STUBS); + TableSchema branchSchema = Mockito.mock(TableSchema.class); + Mockito.when(branchTable.schemaManager().schema(3L)).thenReturn(branchSchema); + Mockito.when(branchSchema.id()).thenReturn(3L); + Mockito.when(branchSchema.fields()).thenReturn(Collections.emptyList()); + Mockito.when(source.getExternalTable()).thenReturn(externalTable); + Mockito.when(source.getPaimonTable()).thenReturn(branchTable); + Mockito.when(source.getCatalog()).thenReturn(catalog); + node.setSource(source); + setField(FileQueryScanNode.class, node, "params", new TFileScanRangeParams()); + + try (MockedStatic paimonUtils = Mockito.mockStatic(PaimonUtils.class)) { + invokePrivateMethod(node, "putHistorySchemaInfo", new Class[] {Long.class}, 3L); + paimonUtils.verify( + () -> PaimonUtils.getSchemaCacheValue(externalTable, 3L), Mockito.never()); + } + + Mockito.verify(branchTable.schemaManager()).schema(3L); + Assert.assertEquals(3L, node.getFileScanRangeParams().getHistorySchemaInfo().get(0).getSchemaId()); + } + private void mockJniReader(PaimonScanNode spyNode) { Mockito.doReturn(false).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonSourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonSourceTest.java new file mode 100644 index 00000000000000..0b4b79ce1e47d3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonSourceTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon.source; + +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.paimon.PaimonExternalTable; + +import com.google.common.collect.ImmutableMap; +import org.apache.paimon.table.Table; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Optional; + +public class PaimonSourceTest { + + @Test + public void testUsesRelationSnapshotInsteadOfStatementCurrentSnapshot() { + TupleDescriptor desc = new TupleDescriptor(new TupleId(1)); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + MvccSnapshot relationSnapshot = Mockito.mock(MvccSnapshot.class); + Table branchTable = Mockito.mock(Table.class); + desc.setTable(externalTable); + Mockito.when(externalTable.getPaimonTable(Optional.of(relationSnapshot))).thenReturn(branchTable); + + PaimonSource source = new PaimonSource(desc, Optional.of(relationSnapshot)); + + Assert.assertSame(branchTable, source.getPaimonTable()); + } + + @Test + public void testBehavioralOptionsStayOnRelationTable() { + TupleDescriptor desc = new TupleDescriptor(new TupleId(1)); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + MvccSnapshot relationSnapshot = Mockito.mock(MvccSnapshot.class); + Table relationTable = Mockito.mock(Table.class); + Table processedTable = Mockito.mock(Table.class); + desc.setTable(externalTable); + Mockito.when(externalTable.getPaimonTable(Optional.of(relationSnapshot))).thenReturn(relationTable); + Mockito.when(relationTable.copy(ImmutableMap.of("scan.plan-sort-partition", "true"))) + .thenReturn(processedTable); + PaimonSource source = new PaimonSource(desc, Optional.of(relationSnapshot)); + TableScanParams params = new TableScanParams(TableScanParams.OPTIONS, + ImmutableMap.of("scan.plan-sort-partition", "true"), java.util.Collections.emptyList()); + + Assert.assertSame(processedTable, source.getPaimonTable(params)); + Mockito.verify(externalTable, Mockito.never()).getPaimonTable(params); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index 7fd9f0120cbe83..a8efc7c3d5040a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -33,6 +33,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.datasource.hive.HMSSchemaCacheValue; import org.apache.doris.datasource.hive.HiveDlaTable; +import org.apache.doris.datasource.mvcc.EmptyMvccSnapshot; import org.apache.doris.nereids.datasets.tpch.AnalyzeCheckTestBase; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; @@ -46,6 +47,7 @@ import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.List; import java.util.Optional; @@ -380,6 +382,12 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { } }; + mockSnapshotAwareSchema(tbl, schema); + mockSnapshotAwareSchema(view1, schema); + mockSnapshotAwareSchema(view2, schema); + mockSnapshotAwareSchema(view3, schema); + mockSnapshotAwareSchema(view4, schema); + db.addTableForTest(tbl); db.addTableForTest(view1); db.addTableForTest(view2); @@ -388,6 +396,13 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { hmsCatalog.addDatabaseForTest(db); } + private void mockSnapshotAwareSchema(HMSExternalTable table, List schema) { + // Binding pins both the snapshot and schema per relation, so HMS mocks must implement + // the same snapshot-aware contract as real tables instead of only the legacy overload. + Mockito.when(table.loadSnapshot(Mockito.any(), Mockito.any())).thenReturn(new EmptyMvccSnapshot()); + Mockito.when(table.getFullSchema(Mockito.any())).thenReturn(schema); + } + @Test public void testQueryView() { SessionVariable sv = connectContext.getSessionVariable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 87d31f533cfa9b..a6e0e861a2ba33 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -175,6 +175,7 @@ public void testPreloadLatestRelationWhenExplicitSnapshotAliasExists() { HMSExternalTable hmsExternalTable = Mockito.mock(HMSExternalTable.class); DatabaseIf database = mockDatabase(); CatalogIf catalog = mockCatalog(); + MvccSnapshot mvccSnapshot = Mockito.mock(MvccSnapshot.class); SessionVariable sessionVariable = new SessionVariable(); sessionVariable.setEnablePreloadExternalMetadata(true); @@ -191,6 +192,9 @@ public void testPreloadLatestRelationWhenExplicitSnapshotAliasExists() { Mockito.when(hmsExternalTable.supportsExternalMetadataPreload()).thenReturn(true); Mockito.when(hmsExternalTable.supportsLatestSnapshotPreload()).thenReturn(true); Mockito.when(hmsExternalTable.getDlaType()).thenReturn(DLAType.HUDI); + // StatementContext stores the returned snapshot, so null is not a valid preload result. + Mockito.when(hmsExternalTable.loadSnapshot(Mockito.>any(), Mockito.any())) + .thenReturn(mvccSnapshot); StatementContext statementContext = new StatementContext(connectContext, new OriginStatement("select 1", 0)); try { @@ -580,6 +584,130 @@ public void testSelectorFreePaimonOptionsPreloadLatestSnapshotBeforeLock() { } } + @Test + public void testLoadSnapshotsKeepsEachRelationSnapshotCurrent() { + ConnectContext connectContext = Mockito.mock(ConnectContext.class); + PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); + DatabaseIf database = mockDatabase(); + CatalogIf catalog = mockCatalog(); + MvccSnapshot firstSnapshot = Mockito.mock(MvccSnapshot.class); + MvccSnapshot secondSnapshot = Mockito.mock(MvccSnapshot.class); + + Mockito.when(table.getName()).thenReturn("historical_table"); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("ctl"); + Mockito.when(table.loadSnapshot(Mockito.>any(), Mockito.any())) + .thenReturn(firstSnapshot, secondSnapshot); + + StatementContext statementContext = new StatementContext(connectContext, new OriginStatement("select 1", 0)); + try { + statementContext.loadSnapshots(table, + Optional.of(new TableSnapshot("1", TableSnapshot.VersionType.VERSION)), Optional.empty()); + org.junit.jupiter.api.Assertions.assertSame(firstSnapshot, + statementContext.getSnapshot(table).orElseThrow(AssertionError::new)); + + statementContext.loadSnapshots(table, + Optional.of(new TableSnapshot("2", TableSnapshot.VersionType.VERSION)), Optional.empty()); + + org.junit.jupiter.api.Assertions.assertSame(secondSnapshot, + statementContext.getSnapshot(table).orElseThrow(AssertionError::new)); + Mockito.verify(table, Mockito.times(2)) + .loadSnapshot(Mockito.>any(), Mockito.any()); + } finally { + statementContext.close(); + } + } + + @Test + public void testLatestSnapshotIsIndependentOfHistoricalRelationOrder() { + ConnectContext connectContext = Mockito.mock(ConnectContext.class); + PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); + DatabaseIf database = mockDatabase(); + CatalogIf catalog = mockCatalog(); + MvccSnapshot latestSnapshot = Mockito.mock(MvccSnapshot.class); + MvccSnapshot historicalSnapshot = Mockito.mock(MvccSnapshot.class); + + Mockito.when(table.getName()).thenReturn("mixed_version_table"); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("ctl"); + Mockito.when(table.loadSnapshot(Mockito.>any(), Mockito.any())) + .thenAnswer(invocation -> ((Optional) invocation.getArgument(0)).isPresent() + ? historicalSnapshot : latestSnapshot); + + StatementContext statementContext = new StatementContext(connectContext, new OriginStatement("select 1", 0)); + try { + statementContext.loadSnapshots(table, Optional.empty(), Optional.empty()); + statementContext.loadSnapshots(table, + Optional.of(new TableSnapshot("1", TableSnapshot.VersionType.VERSION)), Optional.empty()); + org.junit.jupiter.api.Assertions.assertSame(historicalSnapshot, + statementContext.getSnapshot(table).orElseThrow(AssertionError::new)); + + statementContext.loadSnapshots(table, Optional.empty(), Optional.empty()); + + org.junit.jupiter.api.Assertions.assertSame(latestSnapshot, + statementContext.getSnapshot(table).orElseThrow(AssertionError::new)); + Mockito.verify(table, Mockito.times(2)) + .loadSnapshot(Mockito.>any(), Mockito.any()); + } finally { + statementContext.close(); + } + + StatementContext reverseStatementContext = new StatementContext( + connectContext, new OriginStatement("select 1", 0)); + try { + reverseStatementContext.loadSnapshots(table, + Optional.of(new TableSnapshot("1", TableSnapshot.VersionType.VERSION)), Optional.empty()); + org.junit.jupiter.api.Assertions.assertSame(historicalSnapshot, + reverseStatementContext.getSnapshot(table).orElseThrow(AssertionError::new)); + + reverseStatementContext.loadSnapshots(table, Optional.empty(), Optional.empty()); + + org.junit.jupiter.api.Assertions.assertSame(latestSnapshot, + reverseStatementContext.getSnapshot(table).orElseThrow(AssertionError::new)); + Mockito.verify(table, Mockito.times(4)) + .loadSnapshot(Mockito.>any(), Mockito.any()); + } finally { + reverseStatementContext.close(); + } + } + + @Test + public void testInjectedSnapshotRemainsAuthoritativeForLatestRelation() { + ConnectContext connectContext = Mockito.mock(ConnectContext.class); + PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); + DatabaseIf database = mockDatabase(); + CatalogIf catalog = mockCatalog(); + MvccSnapshot injectedSnapshot = Mockito.mock(MvccSnapshot.class); + MvccSnapshot reloadedSnapshot = Mockito.mock(MvccSnapshot.class); + + Mockito.when(table.getName()).thenReturn("mtmv_base_table"); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("ctl"); + Mockito.when(table.loadSnapshot(Mockito.>any(), Mockito.any())) + .thenReturn(reloadedSnapshot); + + StatementContext statementContext = new StatementContext(connectContext, new OriginStatement("select 1", 0)); + try { + statementContext.setSnapshot(new org.apache.doris.datasource.mvcc.MvccTableInfo(table), + injectedSnapshot); + + Optional loaded = statementContext.loadSnapshots( + table, Optional.empty(), Optional.empty()); + + org.junit.jupiter.api.Assertions.assertSame(injectedSnapshot, + loaded.orElseThrow(AssertionError::new)); + Mockito.verify(table, Mockito.never()).loadSnapshot(Mockito.any(), Mockito.any()); + } finally { + statementContext.close(); + } + } + @SuppressWarnings("unchecked") private DatabaseIf mockDatabase() { return Mockito.mock(DatabaseIf.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSinkTest.java new file mode 100644 index 00000000000000..744b7511b99012 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSinkTest.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +public class UnboundIcebergTableSinkTest { + @Test + public void testBranchNameSurvivesPlanCopies() { + Plan child = new LogicalOneRowRelation(new RelationId(1), ImmutableList.of()); + UnboundIcebergTableSink sink = new UnboundIcebergTableSink<>( + ImmutableList.of("catalog", "db", "table"), + ImmutableList.of("old_name"), + ImmutableList.of(), + ImmutableList.of(), + child); + sink = sink.withBranchName(Optional.of("historical_branch")); + + Plan replacementChild = new LogicalOneRowRelation(new RelationId(2), ImmutableList.of()); + UnboundIcebergTableSink copied = (UnboundIcebergTableSink) sink.withChildren( + ImmutableList.of(replacementChild)); + + Assertions.assertEquals(Optional.of("historical_branch"), copied.getBranchName()); + Assertions.assertSame(replacementChild, copied.child()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java index 12ea36c17e87a5..a7c4a4d0184752 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java @@ -51,6 +51,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.List; public class CheckCastTest { @@ -1787,6 +1788,13 @@ public void testCastFromStruct() { StructType structType4 = new StructType(fields4); Assertions.assertTrue(CheckCast.check(structType1, structType4, true)); + StructType requiredString = new StructType(Collections.singletonList( + new StructField("metric", StringType.INSTANCE, false, ""))); + StructType requiredInt = new StructType(Collections.singletonList( + new StructField("metric", IntegerType.INSTANCE, false, ""))); + StructType requiredBigInt = new StructType(Collections.singletonList( + new StructField("metric", BigIntType.INSTANCE, false, ""))); + // Un-strict mode Assertions.assertFalse(CheckCast.check(StructType.SYSTEM_DEFAULT, BooleanType.INSTANCE, false)); Assertions.assertFalse(CheckCast.check(StructType.SYSTEM_DEFAULT, TinyIntType.INSTANCE, false)); @@ -1819,6 +1827,8 @@ public void testCastFromStruct() { Assertions.assertFalse(CheckCast.check(structType1, structType2, false)); Assertions.assertFalse(CheckCast.check(structType1, structType3, false)); Assertions.assertTrue(CheckCast.check(structType1, structType4, false)); + Assertions.assertFalse(CheckCast.check(requiredString, requiredInt, false)); + Assertions.assertTrue(CheckCast.check(requiredInt, requiredBigInt, false)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java index d492100e867f29..13af5fe4b0fcae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java @@ -169,7 +169,7 @@ private LogicalAggregate newNullableFileCountAggregate() { IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); Mockito.when(table.initSelectedPartitions(Mockito.any())) .thenReturn(SelectedPartitions.NOT_PRUNED); - Mockito.when(table.getFullSchema()).thenReturn(ImmutableList.of(nullableColumn)); + Mockito.when(table.getFullSchema(Mockito.any())).thenReturn(ImmutableList.of(nullableColumn)); Mockito.when(table.getName()).thenReturn("nullable_file_table"); CatalogIf catalog = Mockito.mock(CatalogIf.class); Mockito.when(catalog.getName()).thenReturn("catalog"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartitionTest.java new file mode 100644 index 00000000000000..741b2d9ebbec33 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartitionTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +public class PruneFileScanPartitionTest { + @Test + public void testPartitionColumnsUseRelationSnapshot() { + ExternalTable table = Mockito.mock(ExternalTable.class); + LogicalFileScan scan = Mockito.mock(LogicalFileScan.class); + MvccSnapshot relationSnapshot = Mockito.mock(MvccSnapshot.class); + List partitionColumns = Collections.singletonList(new Column("old_partition", Type.INT, true)); + Mockito.when(scan.getRelationSnapshot()).thenReturn(Optional.of(relationSnapshot)); + Mockito.when(table.getPartitionColumns(Optional.of(relationSnapshot))).thenReturn(partitionColumns); + + Assertions.assertSame(partitionColumns, + PruneFileScanPartition.getPartitionColumnsForScan(table, scan)); + Mockito.verify(table).getPartitionColumns(Optional.of(relationSnapshot)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java new file mode 100644 index 00000000000000..c367d8ade27594 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.apache.doris.nereids.rules.expression.check.CheckCast; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class StructLiteralTest { + + @Test + public void testInferFieldNullabilityForRequiredTarget() { + StructLiteral literal = new StructLiteral(ImmutableList.of( + new IntegerLiteral(20), new NullLiteral(StringType.INSTANCE))); + StructType literalType = (StructType) literal.getDataType(); + + Assertions.assertFalse(literalType.getFields().get(0).isNullable()); + Assertions.assertTrue(literalType.getFields().get(1).isNullable()); + + StructType requiredTarget = new StructType(ImmutableList.of( + new StructField("required_metric", BigIntType.INSTANCE, false, ""), + new StructField("required_label", StringType.INSTANCE, true, ""))); + Assertions.assertTrue(CheckCast.check(literalType, requiredTarget, false)); + + StructType nullableTarget = new StructType(ImmutableList.of( + new StructField("metric", BigIntType.INSTANCE, true, ""), + new StructField("label", StringType.INSTANCE, true, ""))); + StructLiteral nonNullLiteral = new StructLiteral(ImmutableList.of( + new IntegerLiteral(10), new StringLiteral("value"))); + Assertions.assertTrue(CheckCast.check(nonNullLiteral.getDataType(), nullableTarget, false)); + Assertions.assertFalse(CheckCast.check(nullableTarget, nonNullLiteral.getDataType(), false)); + } + + @Test + public void testNamedStructInfersValueNullability() { + CreateNamedStruct required = new CreateNamedStruct( + new StringLiteral("metric"), new IntegerLiteral(10)); + StructType requiredType = (StructType) required.customSignature().returnType; + Assertions.assertFalse(requiredType.getFields().get(0).isNullable()); + + CreateNamedStruct nullable = new CreateNamedStruct( + new StringLiteral("metric"), new NullLiteral(BigIntType.INSTANCE)); + StructType nullableType = (StructType) nullable.customSignature().returnType; + Assertions.assertTrue(nullableType.getFields().get(0).isNullable()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java index 8b2f186d1f2ac1..99163c19dcb5be 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutorTest.java @@ -91,14 +91,17 @@ public void testFinalizeSinkAndBeforeExecPropagateRewritableDeleteMetadata() thr Mockito.when(transactionManager.getTransaction(10L)).thenReturn(transaction); IcebergExternalTable table = mockIcebergExternalTable(3, transactionManager); + Table targetIcebergTable = table.getIcebergTable(); NereidsPlanner planner = mockPlanner(Collections.singletonList(scanNode)); ConnectContext ctx = new ConnectContext(); ctx.setQueryId(new TUniqueId(1L, 2L)); ctx.getSessionVariable().setEnableNereidsDistributePlanner(false); ctx.setThreadLocalInfo(); - IcebergDeleteExecutor executor = new IcebergDeleteExecutor(ctx, table, "label", planner, false, -1L); - IcebergDeleteSink sink = new IcebergDeleteSink(table, new org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext()); + IcebergDeleteExecutor executor = new IcebergDeleteExecutor( + ctx, table, targetIcebergTable, "label", planner, false, -1L); + IcebergDeleteSink sink = new IcebergDeleteSink(table, targetIcebergTable, + new org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext()); executor.finalizeSinkForDelete(null, sink, null); TDataSink tDataSink = getTDataSink(sink); @@ -109,7 +112,7 @@ public void testFinalizeSinkAndBeforeExecPropagateRewritableDeleteMetadata() thr executor.txnId = 10L; executor.beforeExec(); - Mockito.verify(transaction).beginDelete(table); + Mockito.verify(transaction).beginDelete(table, targetIcebergTable); ArgumentCaptor>> deleteFilesCaptor = ArgumentCaptor.forClass(Map.class); Mockito.verify(transaction).setRewrittenDeleteFilesByReferencedDataFile(deleteFilesCaptor.capture()); Assertions.assertSame(deleteFile, deleteFilesCaptor.getValue().get(referencedDataFile).get(0)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java index 7ea40b11739835..639069ae29ee78 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutorTest.java @@ -98,8 +98,10 @@ public void testFinalizeSinkAndBeforeExecPropagateRewritableDeleteMetadata() thr ctx.getSessionVariable().setEnableNereidsDistributePlanner(false); ctx.setThreadLocalInfo(); - IcebergMergeExecutor executor = new IcebergMergeExecutor(ctx, table, "label", planner, false, -1L); - IcebergMergeSink sink = new IcebergMergeSink(table, + Table targetIcebergTable = table.getIcebergTable(); + IcebergMergeExecutor executor = new IcebergMergeExecutor( + ctx, table, targetIcebergTable, "label", planner, false, -1L); + IcebergMergeSink sink = new IcebergMergeSink(table, targetIcebergTable, new org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext(), true); executor.finalizeSinkForMerge(null, sink, null); @@ -113,7 +115,7 @@ public void testFinalizeSinkAndBeforeExecPropagateRewritableDeleteMetadata() thr executor.txnId = 11L; executor.beforeExec(); - Mockito.verify(transaction).beginMerge(table); + Mockito.verify(transaction).beginMerge(table, targetIcebergTable); ArgumentCaptor>> deleteFilesCaptor = ArgumentCaptor.forClass(Map.class); Mockito.verify(transaction).setRewrittenDeleteFilesByReferencedDataFile(deleteFilesCaptor.capture()); Assertions.assertSame(deleteFile, deleteFilesCaptor.getValue().get(referencedDataFile).get(0)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java index 98bd8b1f5e92f0..e8940b8dcc791b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java @@ -17,8 +17,33 @@ package org.apache.doris.nereids.trees.plans.commands.insert; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Test for InsertUtils.getFinalErrorMsg() @@ -27,6 +52,52 @@ public class InsertUtilsTest { private static final int MAX_TOTAL_BYTES = 512; + @AfterEach + public void tearDown() { + ConnectContext.remove(); + } + + @Test + public void testNormalizeValuesPinsTargetSnapshotBeforeExpandingDefault() { + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(context, null); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(table.getName()).thenReturn("table"); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergMvccSnapshot snapshot = new IcebergMvccSnapshot(new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(2L, 2L))); + Mockito.when(table.loadSnapshot(Optional.empty(), Optional.empty())).thenReturn(snapshot); + Mockito.when(table.getBaseSchema(false)).thenAnswer(invocation -> { + boolean pinned = statementContext.getSnapshot(table).isPresent(); + String defaultValue = pinned ? "2" : "1"; + return ImmutableList.of(new Column("id", Type.INT, false, null, false, defaultValue, "")); + }); + + UnboundInlineTable values = new UnboundInlineTable(ImmutableList.of(ImmutableList.of())); + UnboundIcebergTableSink sink = new UnboundIcebergTableSink<>( + ImmutableList.of("catalog", "db", "table"), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + values); + + Plan normalized = InsertUtils.normalizePlan(sink, table, Optional.empty(), Optional.empty()); + + LogicalInlineTable normalizedValues = (LogicalInlineTable) normalized.child(0); + List> rows = normalizedValues.getConstantExprsList(); + Assertions.assertEquals("2", rows.get(0).get(0).child(0).toSql()); + Mockito.verify(table, Mockito.times(1)).loadSnapshot(Optional.empty(), Optional.empty()); + } + private String generateString(int length) { return generateString(length, "X"); } @@ -217,4 +288,3 @@ public void testUrlAndFirstErrorMsgSumTooLong_UseUrlPlaceholder() { Assertions.assertTrue(result.length() <= MAX_TOTAL_BYTES); } } - diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java index b8199ea63b8467..2393ca580db785 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java @@ -20,9 +20,19 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; @@ -53,7 +63,7 @@ public void testComputeOutputIncludesInvisibleRowLineageColumnsForIcebergTable() IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); Mockito.when(table.initSelectedPartitions(Mockito.any())).thenReturn(SelectedPartitions.NOT_PRUNED); - Mockito.when(table.getFullSchema()).thenReturn(schema); + Mockito.when(table.getFullSchema(Mockito.any())).thenReturn(schema); Mockito.when(table.getName()).thenReturn("iceberg_tbl"); LogicalFileScan scan = new LogicalFileScan(new RelationId(1), table, @@ -88,4 +98,166 @@ public void testPaimonOptionsBindRelationScopedSnapshotSchema() { scan.computeOutput().stream().map(slot -> slot.getName()).collect(Collectors.toList())); Mockito.verify(table, Mockito.never()).initSelectedPartitions(Mockito.any()); } + + @Test + public void testCapturingRelationSchemaDoesNotAllocateOutputExprIds() throws Exception { + StatementScopeIdGenerator.clear(); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())).thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema(Mockito.any())) + .thenReturn(Collections.singletonList(new Column("id", Type.INT, true))); + Mockito.when(table.getName()).thenReturn("iceberg_tbl"); + + new LogicalFileScan(new RelationId(1), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + + Assertions.assertEquals(new ExprId(10000), StatementScopeIdGenerator.newExprId()); + } + + @Test + public void testHmsIcebergCapturesRelationSnapshotSchema() { + Column historicalColumn = new Column("old_name", Type.INT, true); + Column latestColumn = new Column("new_name", Type.INT, true); + MvccSnapshot historicalSnapshot = Mockito.mock(MvccSnapshot.class); + MvccSnapshot latestSnapshot = Mockito.mock(MvccSnapshot.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())).thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema(Optional.of(historicalSnapshot))) + .thenReturn(Collections.singletonList(historicalColumn)); + Mockito.when(table.getFullSchema(Optional.of(latestSnapshot))) + .thenReturn(Collections.singletonList(latestColumn)); + Mockito.when(table.getName()).thenReturn("hms_iceberg_tbl"); + + LogicalFileScan historicalFirst = new LogicalFileScan(new RelationId(3), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(historicalSnapshot)); + LogicalFileScan latestSecond = new LogicalFileScan(new RelationId(4), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(latestSnapshot)); + LogicalFileScan latestFirst = new LogicalFileScan(new RelationId(5), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(latestSnapshot)); + LogicalFileScan historicalSecond = new LogicalFileScan(new RelationId(6), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(historicalSnapshot)); + + Assertions.assertEquals(Collections.singletonList("old_name"), + historicalFirst.computeOutput().stream().map(slot -> slot.getName()).collect(Collectors.toList())); + Assertions.assertEquals(Collections.singletonList("new_name"), + latestSecond.computeOutput().stream().map(slot -> slot.getName()).collect(Collectors.toList())); + Assertions.assertEquals(Collections.singletonList("new_name"), + latestFirst.computeOutput().stream().map(slot -> slot.getName()).collect(Collectors.toList())); + Assertions.assertEquals(Collections.singletonList("old_name"), + historicalSecond.computeOutput().stream().map(slot -> slot.getName()).collect(Collectors.toList())); + Mockito.verify(table, Mockito.times(2)).getFullSchema(Optional.of(historicalSnapshot)); + Mockito.verify(table, Mockito.times(2)).getFullSchema(Optional.of(latestSnapshot)); + } + + @Test + public void testResolvedSnapshotParticipatesInScanEquality() { + MvccSnapshot firstSnapshot = Mockito.mock(MvccSnapshot.class); + MvccSnapshot secondSnapshot = Mockito.mock(MvccSnapshot.class); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())).thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema(Mockito.any())) + .thenReturn(Collections.singletonList(new Column("id", Type.INT, true))); + Mockito.when(table.getName()).thenReturn("moving_branch_table"); + TableScanParams branch = new TableScanParams(TableScanParams.BRANCH, + Collections.singletonMap(TableScanParams.PARAMS_NAME, "moving"), Collections.emptyList()); + + LogicalFileScan first = new LogicalFileScan(new RelationId(7), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.of(branch), Optional.empty(), + Optional.of(firstSnapshot)); + LogicalFileScan second = new LogicalFileScan(new RelationId(8), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.of(branch), Optional.empty(), + Optional.of(secondSnapshot)); + + Assertions.assertFalse(first.hasSameScanState(second)); + } + + @Test + public void testEquivalentResolvedSnapshotsKeepScanEquality() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())).thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema(Mockito.any())) + .thenReturn(Collections.singletonList(new Column("id", Type.INT, true))); + Mockito.when(table.getName()).thenReturn("fixed_snapshot_table"); + MvccSnapshot firstSnapshot = icebergSnapshot(10L, 3L); + MvccSnapshot secondSnapshot = icebergSnapshot(10L, 3L); + + LogicalFileScan first = new LogicalFileScan(new RelationId(10), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(firstSnapshot)); + LogicalFileScan second = new LogicalFileScan(new RelationId(11), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(secondSnapshot)); + + Assertions.assertTrue(first.hasSameScanState(second)); + } + + @Test + public void testDifferentFrozenNameMappingsBreakScanEquality() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())).thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema(Mockito.any())) + .thenReturn(Collections.singletonList(new Column("id", Type.INT, true))); + Mockito.when(table.getName()).thenReturn("legacy_name_mapping_table"); + MvccSnapshot firstSnapshot = icebergSnapshot(10L, 3L, + Collections.singletonMap(1, Collections.singletonList("legacy_a"))); + MvccSnapshot secondSnapshot = icebergSnapshot(10L, 3L, + Collections.singletonMap(1, Collections.singletonList("legacy_b"))); + + LogicalFileScan first = new LogicalFileScan(new RelationId(12), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(firstSnapshot)); + LogicalFileScan second = new LogicalFileScan(new RelationId(13), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), + Optional.of(secondSnapshot)); + + Assertions.assertFalse(first.hasSameScanState(second)); + } + + private static MvccSnapshot icebergSnapshot(long snapshotId, long schemaId) { + return new IcebergMvccSnapshot(new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(snapshotId, schemaId))); + } + + private static MvccSnapshot icebergSnapshot(long snapshotId, long schemaId, + Map> nameMapping) { + return new IcebergMvccSnapshot(new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(snapshotId, schemaId), Optional.of(nameMapping))); + } + + @Test + public void testHudiCopiesKeepRelationSnapshotAndSchema() { + MvccSnapshot historicalSnapshot = Mockito.mock(MvccSnapshot.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(table.initSelectedPartitions(Optional.of(historicalSnapshot))) + .thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema(Optional.of(historicalSnapshot))) + .thenReturn(Collections.singletonList(new Column("old_name", Type.INT, true))); + Mockito.when(table.getName()).thenReturn("hudi_table"); + + LogicalHudiScan scan = new LogicalHudiScan(new RelationId(9), table, + Collections.singletonList("db"), Collections.emptyList(), Optional.empty(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(historicalSnapshot)); + LogicalHudiScan copy = scan.withCachedOutput(scan.computeOutput()); + + Assertions.assertSame(historicalSnapshot, copy.getRelationSnapshot().orElseThrow(AssertionError::new)); + Assertions.assertEquals(Collections.singletonList("old_name"), + copy.computeOutput().stream().map(Slot::getName).collect(Collectors.toList())); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index 21657a07c64621..b440eaa24210d5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; @@ -42,7 +43,10 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral; import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; @@ -66,6 +70,7 @@ import org.apache.doris.nereids.types.QuantileStateType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; @@ -74,6 +79,7 @@ import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; @@ -179,6 +185,75 @@ public void testVariantCommonTypePreservesLegacyBehavior() { TypeCoercionUtils.findCommonPrimitiveTypeForCaseWhen(v2, anotherV2).get()); } + @Test + public void testCoalesceCastsRequiredStructFieldsToNullableLayout() { + StructType nullableStruct = new StructType(ImmutableList.of( + new StructField("name", VarcharType.createVarcharType(10), true, ""), + new StructField("age", IntegerType.INSTANCE, true, ""))); + SlotReference nullableStructSlot = new SlotReference("value", nullableStruct, true); + StructLiteral requiredStruct = new StructLiteral(ImmutableList.of( + new StringLiteral("Charlie"), new IntegerLiteral(18))); + + Expression coerced = TypeCoercionUtils.processBoundFunction( + new Coalesce(nullableStructSlot, requiredStruct)); + + Assertions.assertInstanceOf(Cast.class, coerced.child(1)); + StructType castType = (StructType) coerced.child(1).getDataType(); + Assertions.assertTrue(castType.getFields().get(0).isNullable()); + Assertions.assertTrue(castType.getFields().get(1).isNullable()); + } + + @Test + public void testLegacyCaseWhenUnionsStructFieldNullabilityInBothOrders() { + StructType required = new StructType(ImmutableList.of( + new StructField("event_time", DateTimeV2Type.of(3), false, ""))); + StructType nullable = new StructType(ImmutableList.of( + new StructField("event_time", DateTimeV2Type.of(6), true, ""))); + + StructType leftRequired = (StructType) TypeCoercionUtils.findWiderCommonTypeForCaseWhen( + ImmutableList.of(required, nullable)).get(); + StructType leftNullable = (StructType) TypeCoercionUtils.findWiderCommonTypeForCaseWhen( + ImmutableList.of(nullable, required)).get(); + + Assertions.assertTrue(leftRequired.getFields().get(0).isNullable()); + Assertions.assertTrue(leftNullable.getFields().get(0).isNullable()); + } + + @Test + public void testLegacySetOperationIncludesCastNullability() { + boolean oldBehavior = GlobalVariable.enableNewTypeCoercionBehavior; + GlobalVariable.enableNewTypeCoercionBehavior = false; + try { + StructType millis = new StructType(ImmutableList.of( + new StructField("event_time", DateTimeV2Type.of(3), false, ""))); + StructType micros = new StructType(ImmutableList.of( + new StructField("event_time", DateTimeV2Type.of(6), false, ""))); + + StructType common = (StructType) org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation + .getAssignmentCompatibleType(millis, micros); + + Assertions.assertTrue(common.getFields().get(0).isNullable()); + } finally { + GlobalVariable.enableNewTypeCoercionBehavior = oldBehavior; + } + } + + @Test + public void testInPredicateStructCastKeepsNullableCastResult() { + StructLiteral stringStruct = new StructLiteral(ImmutableList.of( + new IntegerLiteral(1), new StringLiteral("2"))); + StructLiteral integerStruct = new StructLiteral(ImmutableList.of( + new IntegerLiteral(1), new IntegerLiteral(3))); + InPredicate predicate = new InPredicate(stringStruct, + ImmutableList.of(integerStruct, NullLiteral.INSTANCE)); + + InPredicate coerced = (InPredicate) TypeCoercionUtils.processInPredicate(predicate); + + StructType commonType = (StructType) coerced.getCompareExpr().getDataType(); + Assertions.assertTrue(commonType.getFields().get(1).isNullable(), + "String-to-decimal coercion can produce NULL inside the struct field"); + } + @Test public void testHasCharacterType() { Assertions.assertFalse(TypeCoercionUtils.hasCharacterType(NullType.INSTANCE)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergDeleteSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergDeleteSinkTest.java index 7a9ed79725377e..ef0e6cf3b78a33 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergDeleteSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergDeleteSinkTest.java @@ -45,7 +45,9 @@ public class IcebergDeleteSinkTest { @Test public void testBindDataSinkIncludesRewritableDeleteFileSetsForV3() throws Exception { - IcebergDeleteSink sink = new IcebergDeleteSink(mockIcebergExternalTable(3), new DeleteCommandContext()); + IcebergExternalTable targetTable = mockIcebergExternalTable(3); + IcebergDeleteSink sink = new IcebergDeleteSink( + targetTable, targetTable.getIcebergTable(), new DeleteCommandContext()); sink.setRewritableDeleteFileSets(Collections.singletonList(buildDeleteFileSet())); sink.bindDataSink(Optional.empty()); @@ -59,7 +61,9 @@ public void testBindDataSinkIncludesRewritableDeleteFileSetsForV3() throws Excep @Test public void testBindDataSinkSkipsRewritableDeleteFileSetsForV2() throws Exception { - IcebergDeleteSink sink = new IcebergDeleteSink(mockIcebergExternalTable(2), new DeleteCommandContext()); + IcebergExternalTable targetTable = mockIcebergExternalTable(2); + IcebergDeleteSink sink = new IcebergDeleteSink( + targetTable, targetTable.getIcebergTable(), new DeleteCommandContext()); sink.setRewritableDeleteFileSets(Collections.singletonList(buildDeleteFileSet())); sink.bindDataSink(Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java new file mode 100644 index 00000000000000..334c70880e2738 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergTableSinkTest.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext; +import org.apache.doris.thrift.TIcebergTableSink; + +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +public class IcebergTableSinkTest { + @Test + public void testBindUsesPinnedIcebergTableMetadata() throws Exception { + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); + Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); + Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(null); + Mockito.when(catalogProperty.getStoragePropertiesMap()).thenReturn(Collections.emptyMap()); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(targetTable.isView()).thenReturn(false); + Mockito.when(targetTable.getCatalog()).thenReturn(catalog); + Mockito.when(targetTable.getDbName()).thenReturn("db"); + Mockito.when(targetTable.getName()).thenReturn("table"); + + Schema pinnedSchema = new Schema(1, + Types.NestedField.required(1, "pinned_id", Types.IntegerType.get())); + Table pinnedTable = mockTable(pinnedSchema); + Schema liveSchema = new Schema(2, + Types.NestedField.required(2, "live_id", Types.IntegerType.get())); + Table liveTable = mockTable(liveSchema); + Mockito.when(targetTable.getIcebergTable()).thenReturn(liveTable); + + IcebergTableSink sink = new IcebergTableSink(targetTable, pinnedTable); + sink.bindDataSink(Optional.empty()); + + TIcebergTableSink thriftSink = sink.tDataSink.getIcebergTableSink(); + Assertions.assertTrue(thriftSink.getSchemaJson().contains("pinned_id")); + Assertions.assertFalse(thriftSink.getSchemaJson().contains("live_id")); + } + + @Test + public void testRewriteKeepsPinnedFormatVersionForRowLineageSchema() throws Exception { + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); + Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); + Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(null); + Mockito.when(catalogProperty.getStoragePropertiesMap()).thenReturn(Collections.emptyMap()); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(targetTable.isView()).thenReturn(false); + Mockito.when(targetTable.getCatalog()).thenReturn(catalog); + Mockito.when(targetTable.getDbName()).thenReturn("db"); + Mockito.when(targetTable.getName()).thenReturn("table"); + + Schema schema = new Schema(1, + Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table pinnedV3Table = mockTable(schema, + Collections.singletonMap(TableProperties.FORMAT_VERSION, "3")); + Table refreshedV2Table = mockTable(schema, + Collections.singletonMap(TableProperties.FORMAT_VERSION, "2")); + Mockito.when(targetTable.getIcebergTable()).thenReturn(pinnedV3Table, refreshedV2Table); + + IcebergInsertCommandContext insertContext = new IcebergInsertCommandContext(); + insertContext.setRewriting(true); + IcebergTableSink sink = new IcebergTableSink(targetTable); + sink.bindDataSink(Optional.of(insertContext)); + + String schemaJson = sink.tDataSink.getIcebergTableSink().getSchemaJson(); + Assertions.assertTrue(schemaJson.contains("_row_id")); + Assertions.assertTrue(schemaJson.contains("_last_updated_sequence_number")); + Mockito.verify(targetTable, Mockito.times(1)).getIcebergTable(); + } + + private static Table mockTable(Schema schema) { + return mockTable(schema, Collections.emptyMap()); + } + + private static Table mockTable(Schema schema, Map properties) { + Table table = Mockito.mock(Table.class); + PartitionSpec spec = PartitionSpec.unpartitioned(); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(spec); + Mockito.when(table.specs()).thenReturn(Collections.singletonMap(spec.specId(), spec)); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(properties); + Mockito.when(table.location()).thenReturn("file:///tmp/iceberg-table"); + Mockito.when(table.name()).thenReturn("table"); + return table; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index 9c3274b00e8d29..c54ae15ca2bc5a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.datasource.hive.HiveDlaTable; import org.apache.doris.datasource.hive.source.HiveScanNode; +import org.apache.doris.datasource.mvcc.EmptyMvccSnapshot; import org.apache.doris.datasource.systable.PartitionsSysTable; import org.apache.doris.nereids.datasets.tpch.AnalyzeCheckTestBase; import org.apache.doris.nereids.parser.NereidsParser; @@ -215,6 +216,11 @@ private void init(HMSExternalCatalog hmsCatalog) { Mockito.when(view2.getDatabase()).thenReturn(db); Mockito.when(view2.getSupportedSysTables()).thenReturn(PartitionsSysTable.HIVE_SUPPORTED_SYS_TABLES); + mockSnapshotAwareSchema(tbl, schema); + mockSnapshotAwareSchema(tbl2, schema); + mockSnapshotAwareSchema(view1, schema); + mockSnapshotAwareSchema(view2, schema); + db.addTableForTest(tbl); db.addTableForTest(tbl2); db.addTableForTest(view1); @@ -231,6 +237,13 @@ private void init(HMSExternalCatalog hmsCatalog) { olapScanNode = new OlapScanNode(new PlanNodeId(1), desc, "tb1ScanNode", ScanContext.EMPTY); } + private void mockSnapshotAwareSchema(HMSExternalTable table, List schema) { + // Binding pins both the snapshot and schema per relation, so HMS mocks must implement + // the same snapshot-aware contract as real tables instead of only the legacy overload. + Mockito.when(table.loadSnapshot(Mockito.any(), Mockito.any())).thenReturn(new EmptyMvccSnapshot()); + Mockito.when(table.getFullSchema(Mockito.any())).thenReturn(schema); + } + @Test public void testHitSqlCacheByNereids() { init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); diff --git a/regression-test/data/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.out b/regression-test/data/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.out index 87a878776933de..2ecac1e7007dec 100644 --- a/regression-test/data/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.out +++ b/regression-test/data/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.out @@ -33,4 +33,3 @@ col3 int Yes true \N -- !b4_new_schema -- 1 \N - diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_branch_tag_operate.out b/regression-test/data/external_table_p0/iceberg/iceberg_branch_tag_operate.out index 9af28fc98f617d..f4904c1962e10b 100644 --- a/regression-test/data/external_table_p0/iceberg/iceberg_branch_tag_operate.out +++ b/regression-test/data/external_table_p0/iceberg/iceberg_branch_tag_operate.out @@ -170,4 +170,3 @@ 1 a 1.0 2 b 2.0 3 c 3.0 - diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_query_tag_branch.out b/regression-test/data/external_table_p0/iceberg/iceberg_query_tag_branch.out index 49eb7ae72b5a79..e98c451232fcc0 100644 --- a/regression-test/data/external_table_p0/iceberg/iceberg_query_tag_branch.out +++ b/regression-test/data/external_table_p0/iceberg/iceberg_query_tag_branch.out @@ -6,7 +6,7 @@ 1 -- !branch_3 -- -1 \N \N +1 -- !branch_4 -- 1 \N \N @@ -17,8 +17,8 @@ 2 -- !branch_6 -- -1 \N \N -2 \N \N +1 +2 -- !branch_6 -- 1 \N \N @@ -31,9 +31,9 @@ 3 -- !branch_9 -- -1 \N \N -2 \N \N -3 4 \N +1 \N +2 \N +3 4 -- !branch_10 -- 1 \N \N @@ -42,19 +42,19 @@ 1 -- !branch_12 -- -1 \N \N +1 -- !branch_13 -- 1 \N \N 2 \N \N -- !branch_14 -- -1 \N -2 \N +1 +2 -- !branch_15 -- -1 \N \N -2 \N \N +1 +2 -- !branch_16 -- 1 \N \N @@ -67,9 +67,9 @@ 3 4 -- !branch_18 -- -1 \N \N -2 \N \N -3 4 \N +1 \N +2 \N +3 4 -- !tag_1 -- 1 @@ -128,15 +128,15 @@ 1 \N \N -- !version_2 -- -1 \N \N +1 -- !version_3 -- 1 \N \N 2 \N \N -- !version_4 -- -1 \N \N -2 \N \N +1 +2 -- !version_5 -- 1 \N \N @@ -144,9 +144,9 @@ 3 4 \N -- !version_6 -- -1 \N \N -2 \N \N -3 4 \N +1 \N +2 \N +3 4 -- !version_7 -- 1 @@ -196,22 +196,22 @@ 3 -- !sub_join_branch_with_branch_1 -- -1 \N \N 1 \N \N +1 1 -- !sub_join_branch_with_branch_2 -- -1 \N \N 1 \N \N +1 1 -- !sub_join_branch_with_branch_3 -- -1 \N \N 1 \N \N -2 \N \N 2 \N \N +1 1 +2 2 -- !sub_join_branch_with_branch_4 -- -1 \N \N 1 \N \N +1 1 -- !sub_join_branch_with_branch_5 -- -1 \N \N 1 \N \N -2 \N \N 2 \N \N -3 4 \N 3 4 \N +1 \N 1 \N +2 \N 2 \N +3 4 3 4 -- !sub_join_tag_with_tag_1 -- 1 1 @@ -239,17 +239,17 @@ 3 4 3 4 -- !sub_with_branch_1 -- -1 \N +1 -- !sub_with_branch_2 -- -2 \N +2 -- !sub_with_branch_3 -- -3 4 \N +3 4 -- !sub_with_branch_4 -- -2 \N \N -3 4 \N +2 \N +3 4 -- !sub_with_tag_1 -- 1 @@ -271,7 +271,7 @@ 1 -- !branch_3 -- -1 \N \N +1 -- !branch_4 -- 1 \N \N @@ -282,8 +282,8 @@ 2 -- !branch_6 -- -1 \N \N -2 \N \N +1 +2 -- !branch_6 -- 1 \N \N @@ -296,9 +296,9 @@ 3 -- !branch_9 -- -1 \N \N -2 \N \N -3 4 \N +1 \N +2 \N +3 4 -- !branch_10 -- 1 \N \N @@ -307,19 +307,19 @@ 1 -- !branch_12 -- -1 \N \N +1 -- !branch_13 -- 1 \N \N 2 \N \N -- !branch_14 -- -1 \N -2 \N +1 +2 -- !branch_15 -- -1 \N \N -2 \N \N +1 +2 -- !branch_16 -- 1 \N \N @@ -332,9 +332,9 @@ 3 4 -- !branch_18 -- -1 \N \N -2 \N \N -3 4 \N +1 \N +2 \N +3 4 -- !tag_1 -- 1 @@ -393,15 +393,15 @@ 1 \N \N -- !version_2 -- -1 \N \N +1 -- !version_3 -- 1 \N \N 2 \N \N -- !version_4 -- -1 \N \N -2 \N \N +1 +2 -- !version_5 -- 1 \N \N @@ -409,9 +409,9 @@ 3 4 \N -- !version_6 -- -1 \N \N -2 \N \N -3 4 \N +1 \N +2 \N +3 4 -- !version_7 -- 1 @@ -461,22 +461,22 @@ 3 -- !sub_join_branch_with_branch_1 -- -1 \N \N 1 \N \N +1 1 -- !sub_join_branch_with_branch_2 -- -1 \N \N 1 \N \N +1 1 -- !sub_join_branch_with_branch_3 -- -1 \N \N 1 \N \N -2 \N \N 2 \N \N +1 1 +2 2 -- !sub_join_branch_with_branch_4 -- -1 \N \N 1 \N \N +1 1 -- !sub_join_branch_with_branch_5 -- -1 \N \N 1 \N \N -2 \N \N 2 \N \N -3 4 \N 3 4 \N +1 \N 1 \N +2 \N 2 \N +3 4 3 4 -- !sub_join_tag_with_tag_1 -- 1 1 @@ -504,17 +504,17 @@ 3 4 3 4 -- !sub_with_branch_1 -- -1 \N +1 -- !sub_with_branch_2 -- -2 \N +2 -- !sub_with_branch_3 -- -3 4 \N +3 4 -- !sub_with_branch_4 -- -2 \N \N -3 4 \N +2 \N +3 4 -- !sub_with_tag_1 -- 1 @@ -528,4 +528,3 @@ -- !sub_with_tag_4 -- 2 \N 3 4 - diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.out b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.out index 7546005e2f2381..cf3ccac5bc8ac7 100644 --- a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.out +++ b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.out @@ -329,4 +329,3 @@ phone text Yes true \N 5 Eve 91.3 eve@example.com \N 6 Frank 88.7 frank@example.com \N 7 Grace 93.2 grace@example.com 555-0123 - diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.out new file mode 100644 index 00000000000000..27bcd3676acd8f --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.out @@ -0,0 +1,35 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !old_snapshot -- +1 old-1 10 + +-- !new_snapshot -- +1 old-1 10 +2 new-2 20 + +-- !historical_then_latest -- +1 old-1 old-1 + +-- !latest_then_historical -- +1 old-1 old-1 + +-- !historical_join -- +1 old-1 old-1 + +-- !reverse_historical_join -- +1 old-1 old-1 + +-- !historical_union -- +1 old-1 +1 old-1 +2 new-2 + +-- !nested_historical_union -- +1 10 +1 10 +2 20 + +-- !historical_cte -- +1 old-1 old-1 + +-- !historical_correlated_subquery -- +1 old-1 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.out new file mode 100644 index 00000000000000..f7c54341540a1c --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rollback_old_snapshot -- +1 old-1 10 + +-- !rollback_new_snapshot -- +1 old-1 10 +2 new-2 20 + +-- !rollback_current_after_snapshot -- +1 old-1 10 + +-- !rollback_old_tag -- +1 old-1 10 + +-- !rollback_new_tag -- +1 old-1 10 +2 new-2 20 + +-- !cherrypick_current -- +1 old-1 10 +2 new-2 20 + +-- !cherrypick_old_tag -- +1 old-1 10 + +-- !timestamp_current -- +1 old-1 + +-- !timestamp_old_snapshot -- +1 old-1 + +-- !pre_fast_forward_branch -- +1 old-1 10 + +-- !pre_fast_forward_tag -- +1 old-1 10 + +-- !post_fast_forward_branch -- +1 old-1 10 +2 new-2 6000000000 + +-- !post_fast_forward_tag -- +1 old-1 10 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.out new file mode 100644 index 00000000000000..eebb14076e43bb --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !doris_nested_branch_current_schema -- +1 10 \N 100 \N 1000 \N + +-- !top_branch_current_schema -- +1 alpha \N 10 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_sys_table.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_sys_table.out index 16a7e4b0aab53d9a269878187d8e4cbb28cd3f56..6a5298b356f4988dce860dc3f231bd61899664b2 100644 GIT binary patch delta 476 zcmeA@!t~@Q(}a1-3VHb@3VEeDIXY>XIjQk!`9-;jB~~S=6+ls_%*6EyoCwZ&DFkP; zLeNC!$yF?mC^v>}vp`Q2)8_NywwyrXv_6t0{8KBL5ZuZ9Q$>+@jVflF zvw^B=aG&TDr6%VWrNk%am*$mNC1s{(<{|l+Y;RIv$$KY+$2Jq;>oms64cVHL4|{h_ R{#P$M+1Dp*bNsBe>;T}-v#S6A delta 264 zcmaEIl&SX!(}a0~I%%0Xsqty~MY)M3RwbzwB@+)SOg_UZwfQ~kMCQr!SRUb1@!4_n zYxhc~$*0RZ?~Y*GLK diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.out b/regression-test/data/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.out new file mode 100644 index 00000000000000..38b5ce7e00e915 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !independent_branch_schema -- +1 base-1 10 \N +2 branch-2 6000000000 branch-only-2 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.out b/regression-test/data/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.out new file mode 100644 index 00000000000000..9ae2aab6a64b68 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !old_snapshot -- +1 old-1 10 + +-- !new_snapshot -- +1 old-1 10 +2 new-2 20 + +-- !historical_join -- +1 old-1 old-1 + +-- !reverse_historical_join -- +1 old-1 old-1 + +-- !historical_union -- +1 old-1 +1 old-1 +2 new-2 + +-- !nested_historical_union -- +1 10 +1 10 +2 20 + +-- !historical_cte -- +1 old-1 old-1 + +-- !historical_correlated_subquery -- +1 old-1 diff --git a/regression-test/suites/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.groovy b/regression-test/suites/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.groovy index 48de78fd282842..88c782dc779f3c 100644 --- a/regression-test/suites/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.groovy +++ b/regression-test/suites/external_table_p0/iceberg/branch_tag/iceberg_branch_tag_schema_change_extended.groovy @@ -57,7 +57,8 @@ suite("iceberg_branch_tag_schema_change_extended", "p0,external,doris,external_d // Test 3.1.2: Drop column after branch query sql """ alter table ${table_name} create branch b2_schema """ sql """ alter table ${table_name} drop column name """ - qt_b2_no_dropped_col """select * from ${table_name}@branch(b2_schema) order by id """ // Should not have 'name' column + // Named branches resolve rows at the branch head but use the table's current schema. + qt_b2_no_dropped_col """select * from ${table_name}@branch(b2_schema) order by id """ // Recreate table for next tests sql """ drop table if exists ${table_name} """ @@ -69,8 +70,9 @@ suite("iceberg_branch_tag_schema_change_extended", "p0,external,doris,external_d sql """ alter table ${table_name} modify column id bigint """ qt_b3_new_type """ select * from ${table_name}@branch(b3_schema) where id = 1 """ // Should use new type - // Test 3.1.4: Branch write with new schema + // Test 3.1.4: Branch writes use the shared latest table schema sql """ alter table ${table_name} add column new_col string """ + // Iceberg branch commits advance the branch to a snapshot written with current table metadata. sql """ insert into ${table_name}@branch(b3_schema)(id, value, new_col) values (3, 30, 'test') """ qt_b3_with_new_col """ select * from ${table_name}@branch(b3_schema) where id = 3 """ @@ -134,4 +136,3 @@ suite("iceberg_branch_tag_schema_change_extended", "p0,external,doris,external_d qt_b4_new_schema """ select * from ${table_name}@branch(b4_schema) where id = 1 """ // Should have new_col } - diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy index c5afdf93bedfed..302b21173d8fc6 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy @@ -218,14 +218,14 @@ suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_ // test branch/tag with schema change qt_sc01 """select * from tmp_schema_change_branch order by id;""" - /// select by branch will use table schema + // A named branch fixes the rows at its head but still exposes the table's current schema. qt_sc02 """select * from tmp_schema_change_branch@branch(test_branch) order by id;;""" qt_sc03 """select * from tmp_schema_change_branch for version as of "test_branch" order by id;;""" List> refs = sql """select * from tmp_schema_change_branch\$refs order by name""" String s_main = refs.get(0)[2] String s_test_branch = refs.get(1)[2] - /// select by version will use branch schema + // A numeric snapshot keeps the historical schema recorded by that snapshot. qt_sc04 """SELECT * FROM tmp_schema_change_branch for VERSION AS OF ${s_test_branch} order by id;""" qt_sc05 """SELECT * FROM tmp_schema_change_branch for VERSION AS OF ${s_main} order by id;""" diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy index 8e70003f1c7629..8d7991e76addf9 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_query_tag_branch.groovy @@ -48,25 +48,26 @@ suite("iceberg_query_tag_branch", "p0,external,doris,external_docker,external_do def query_tag_branch_only = { + // Explicit projections must use the schema pinned by each historical reference. qt_branch_1 """ select * from tag_branch_table@branch(b1) order by c1;""" qt_branch_2 """ select c1 from tag_branch_table@branch(b1) order by c1;""" - qt_branch_3 """ select c1,c2,c3 from tag_branch_table@branch(b1) order by c1;""" + qt_branch_3 """ select c1 from tag_branch_table@branch(b1) order by c1;""" qt_branch_4 """ select * from tag_branch_table@branch(b2) order by c1 ;""" qt_branch_5 """ select c1 from tag_branch_table@branch(b2) order by c1;""" - qt_branch_6 """ select c1,c2,c3 from tag_branch_table@branch(b2) order by c1;""" + qt_branch_6 """ select c1 from tag_branch_table@branch(b2) order by c1;""" qt_branch_6 """ select * from tag_branch_table@branch(b3) order by c1 ;""" qt_branch_7 """ select c1 from tag_branch_table@branch(b3) order by c1;""" - qt_branch_9 """ select c1,c2,c3 from tag_branch_table@branch(b3) order by c1;""" + qt_branch_9 """ select c1,c2 from tag_branch_table@branch(b3) order by c1;""" qt_branch_10 """ select * from tag_branch_table@branch('name'='b1') order by c1 ;""" qt_branch_11 """ select c1 from tag_branch_table@branch('name'='b1') order by c1 ;""" - qt_branch_12 """ select c1,c2,c3 from tag_branch_table@branch(b1) order by c1;""" + qt_branch_12 """ select c1 from tag_branch_table@branch(b1) order by c1;""" qt_branch_13 """ select * from tag_branch_table@branch('name'='b2') order by c1 ;""" - qt_branch_14 """ select c1,c2 from tag_branch_table@branch('name'='b2') order by c1 ;""" - qt_branch_15 """ select c1,c2,c3 from tag_branch_table@branch(b2) order by c1;""" + qt_branch_14 """ select c1 from tag_branch_table@branch('name'='b2') order by c1 ;""" + qt_branch_15 """ select c1 from tag_branch_table@branch(b2) order by c1;""" qt_branch_16 """ select * from tag_branch_table@branch('name'='b3') order by c1 ;""" qt_branch_17 """ select c1,c2 from tag_branch_table@branch('name'='b3') order by c1 ;""" - qt_branch_18 """ select c1,c2,c3 from tag_branch_table@branch(b3) order by c1;""" + qt_branch_18 """ select c1,c2 from tag_branch_table@branch(b3) order by c1;""" qt_tag_1 """ select * from tag_branch_table@tag(t1) order by c1 ;""" qt_tag_2 """ select c1 from tag_branch_table@tag(t1) order by c1 ;""" @@ -84,11 +85,11 @@ suite("iceberg_query_tag_branch", "p0,external,doris,external_docker,external_do qt_tag_13 """ select c1,c2 from tag_branch_table@tag('name'='t3') order by c1 """ qt_version_1 """ select * from tag_branch_table for version as of 'b1' order by c1 ;""" - qt_version_2 """ select c1,c2,c3 from tag_branch_table for version as of 'b1' order by c1 ;""" + qt_version_2 """ select c1 from tag_branch_table for version as of 'b1' order by c1 ;""" qt_version_3 """ select * from tag_branch_table for version as of 'b2' order by c1 ;""" - qt_version_4 """ select c1,c2,c3 from tag_branch_table for version as of 'b2' order by c1 ;""" + qt_version_4 """ select c1 from tag_branch_table for version as of 'b2' order by c1 ;""" qt_version_5 """ select * from tag_branch_table for version as of 'b3' order by c1 ;""" - qt_version_6 """ select c1,c2,c3 from tag_branch_table for version as of 'b3' order by c1 ;""" + qt_version_6 """ select c1,c2 from tag_branch_table for version as of 'b3' order by c1 ;""" qt_version_7 """ select * from tag_branch_table for version as of 't1' order by c1 ;""" qt_version_8 """ select c1 from tag_branch_table for version as of 't1' order by c1 ;""" @@ -108,23 +109,23 @@ suite("iceberg_query_tag_branch", "p0,external,doris,external_docker,external_do } def query_tag_branch_in_subquery = { - qt_sub_join_branch_with_branch_1 """ SELECT t1.c1, t1.c2, t1.c3, t2.c1, t2.c2, t2.c3 + qt_sub_join_branch_with_branch_1 """ SELECT t1.c1, t2.c1 FROM tag_branch_table@branch(b1) t1 JOIN tag_branch_table@branch(b2) t2 ON t1.c1 = t2.c1 order by t1.c1; """ - qt_sub_join_branch_with_branch_2 """ SELECT t1.c1, t1.c2, t1.c3, t2.c1, t2.c2, t2.c3 + qt_sub_join_branch_with_branch_2 """ SELECT t1.c1, t2.c1 FROM tag_branch_table@branch(b1) t1 JOIN tag_branch_table@branch(b3) t2 ON t1.c1 = t2.c1 order by t1.c1; """ - qt_sub_join_branch_with_branch_3 """ SELECT t1.c1, t1.c2, t1.c3, t2.c1, t2.c2, t2.c3 + qt_sub_join_branch_with_branch_3 """ SELECT t1.c1, t2.c1 FROM tag_branch_table@branch(b2) t1 JOIN tag_branch_table@branch(b3) t2 ON t1.c1 = t2.c1 order by t1.c1; """ - qt_sub_join_branch_with_branch_4 """ SELECT t1.c1, t1.c2, t1.c3, t2.c1, t2.c2, t2.c3 + qt_sub_join_branch_with_branch_4 """ SELECT t1.c1, t2.c1 FROM tag_branch_table@branch(b1) t1 JOIN tag_branch_table@branch(b1) t2 ON t1.c1 = t2.c1 order by t1.c1; """ - qt_sub_join_branch_with_branch_5 """ SELECT t1.c1, t1.c2, t1.c3, t2.c1, t2.c2, t2.c3 + qt_sub_join_branch_with_branch_5 """ SELECT t1.c1, t1.c2, t2.c1, t2.c2 FROM tag_branch_table@branch(b3) t1 JOIN tag_branch_table@branch(b3) t2 ON t1.c1 = t2.c1 order by t1.c1; """ @@ -164,10 +165,10 @@ suite("iceberg_query_tag_branch", "p0,external,doris,external_docker,external_do WHERE t1.c1 > 1 order by t1.c1; """ - qt_sub_with_branch_1 """ WITH t1 AS ( SELECT c1,c2 FROM tag_branch_table@branch(b1) WHERE c1 > 0) SELECT * FROM t1 order by c1; """ - qt_sub_with_branch_2 """ WITH t1 AS ( SELECT c1,c2 FROM tag_branch_table@branch(b2) WHERE c1 > 1) SELECT * FROM t1 order by c1; """ - qt_sub_with_branch_3 """ WITH t1 AS ( SELECT c1,c2,c3 FROM tag_branch_table@branch(b3) WHERE c2 IS NOT NULL) SELECT * FROM t1 order by c1; """ - qt_sub_with_branch_4 """ WITH t1 AS ( SELECT c1,c2,c3 FROM tag_branch_table@branch(b3) WHERE c1 > 1) SELECT * FROM t1 order by c1; """ + qt_sub_with_branch_1 """ WITH t1 AS ( SELECT c1 FROM tag_branch_table@branch(b1) WHERE c1 > 0) SELECT * FROM t1 order by c1; """ + qt_sub_with_branch_2 """ WITH t1 AS ( SELECT c1 FROM tag_branch_table@branch(b2) WHERE c1 > 1) SELECT * FROM t1 order by c1; """ + qt_sub_with_branch_3 """ WITH t1 AS ( SELECT c1,c2 FROM tag_branch_table@branch(b3) WHERE c2 IS NOT NULL) SELECT * FROM t1 order by c1; """ + qt_sub_with_branch_4 """ WITH t1 AS ( SELECT c1,c2 FROM tag_branch_table@branch(b3) WHERE c1 > 1) SELECT * FROM t1 order by c1; """ qt_sub_with_tag_1 """ WITH t1 AS ( SELECT c1 FROM tag_branch_table@tag(t1) WHERE c1 > 0) SELECT * FROM t1 order by c1; """ qt_sub_with_tag_2 """ WITH t1 AS ( SELECT c1 FROM tag_branch_table@tag(t2) WHERE c1 > 1) SELECT * FROM t1 order by c1; """ diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.groovy index d0a1c9144c04b6..047b75cf33e605 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl_with_branch.groovy @@ -150,43 +150,42 @@ suite("iceberg_schema_change_ddl_with_branch", "p0,external,doris,external_docke qt_tag5_final """ SELECT * FROM ${branch_table_name}@tag(tag5) ORDER BY id """ // ================================================================================== - // Verify schema behavior: branches get latest schema, tags keep creation-time schema + // Verify schema behavior: branches use the current table schema, tags keep creation-time schema // ================================================================================== // Test specific column queries to verify schema differences - // IMPORTANT: Branches will use the LATEST schema from main branch - // Tags will use the schema from when they were created + // Branches use the current table schema; tags use the schema from when they were created. - // branch1: should use LATEST schema (same as main) - id, name, grade, email, phone + // branch1 uses the current table schema - id, name, grade, email, phone qt_branch1_latest_schema """ SELECT * FROM ${branch_table_name}@branch(branch1) ORDER BY id """ // tag1: should have ORIGINAL schema when created - id, name, age, score (no email, phone, grade) qt_tag1_original_schema """ SELECT * FROM ${branch_table_name}@tag(tag1) ORDER BY id """ qt_tag1_age_score """ SELECT id, age, score FROM ${branch_table_name}@tag(tag1) ORDER BY id """ - // branch2: should use LATEST schema (same as main) - id, name, grade, email, phone + // branch2 uses the current table schema - id, name, grade, email, phone qt_branch2_latest_schema """ SELECT * FROM ${branch_table_name}@branch(branch2) ORDER BY id """ // tag2: should have schema when created - id, name, age, score, email (no phone, grade) qt_tag2_creation_schema """ SELECT * FROM ${branch_table_name}@tag(tag2) ORDER BY id """ qt_tag2_age_email """ SELECT id, age, score, email FROM ${branch_table_name}@tag(tag2) ORDER BY id """ - // branch3: should use LATEST schema (same as main) - id, name, grade, email, phone + // branch3 uses the current table schema - id, name, grade, email, phone qt_branch3_latest_schema """ SELECT * FROM ${branch_table_name}@branch(branch3) ORDER BY id """ // tag3: should have schema when created - id, name, score, email (no age, phone, grade) qt_tag3_creation_schema """ SELECT * FROM ${branch_table_name}@tag(tag3) ORDER BY id """ qt_tag3_score_email """ SELECT id, score, email FROM ${branch_table_name}@tag(tag3) ORDER BY id """ - // branch4: should use LATEST schema (same as main) - id, name, grade, email, phone + // branch4 uses the current table schema - id, name, grade, email, phone qt_branch4_latest_schema """ SELECT * FROM ${branch_table_name}@branch(branch4) ORDER BY id """ // tag4: should have schema when created - id, name, grade, email (no age, score, phone) qt_tag4_creation_schema """ SELECT * FROM ${branch_table_name}@tag(tag4) ORDER BY id """ qt_tag4_grade_email """ SELECT id, grade, email FROM ${branch_table_name}@tag(tag4) ORDER BY id """ - // branch5: should use LATEST schema (same as main) - id, name, grade, email, phone + // branch5 uses the current table schema - id, name, grade, email, phone qt_branch5_latest_schema """ SELECT * FROM ${branch_table_name}@branch(branch5) ORDER BY id """ // tag5: should have schema when created - id, name, grade, email, phone @@ -197,14 +196,13 @@ suite("iceberg_schema_change_ddl_with_branch", "p0,external,doris,external_docke // Negative tests: verify schema behavior differences between branches and tags // ================================================================================== - // ALL BRANCHES should have the LATEST schema (same as main) - // So all branches should have: id, name, grade, email, phone - + // All branches expose the current table columns: id, name, grade, email, phone. + // Verify all branches have the latest columns qt_all_branches_have_grade """ SELECT id, grade FROM ${branch_table_name}@branch(branch1) WHERE grade > 0 ORDER BY id """ qt_all_branches_have_email """ SELECT id, email FROM ${branch_table_name}@branch(branch2) WHERE email IS NOT NULL ORDER BY id """ qt_all_branches_have_phone """ SELECT id, phone FROM ${branch_table_name}@branch(branch3) WHERE phone IS NOT NULL ORDER BY id """ - + // All branches should NOT have old columns that were dropped/renamed test { sql """ SELECT age FROM ${branch_table_name}@branch(branch1) """ @@ -276,7 +274,7 @@ suite("iceberg_schema_change_ddl_with_branch", "p0,external,doris,external_docke // Main branch has the latest schema qt_summary_main """ SELECT * FROM ${branch_table_name} ORDER BY id """ - // ALL BRANCHES use the LATEST schema (same as main) + // All branches use the current table schema. qt_summary_branch1 """ SELECT * FROM ${branch_table_name}@branch(branch1) ORDER BY id """ qt_summary_branch2 """ SELECT * FROM ${branch_table_name}@branch(branch2) ORDER BY id """ qt_summary_branch3 """ SELECT * FROM ${branch_table_name}@branch(branch3) ORDER BY id """ diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.groovy index a9fcfb4aeb7fa7..cd909c3b46740f 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.groovy @@ -82,21 +82,35 @@ suite("test_iceberg_schema_dual_relation_matrix", sql """use ${dbName}""" // Scenario TC07-baseline: each historical relation resolves its own schema in isolation. - assertEquals([[1, "old-1", 10]], sql(""" + qt_old_snapshot """ select id, old_name, info.added from ${tableName} for version as of ${oldSnapshot} order by id - """)) - assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + """ + qt_new_snapshot """ select id, new_name, info.renamed from ${tableName} for version as of ${newSnapshot} order by id - """)) + """ + + // Latest metadata must be restored even when a historical relation is bound first. + qt_historical_then_latest """ + select h.id, h.old_name, l.new_name + from ${tableName} for version as of ${oldSnapshot} h + join ${tableName} l on h.id = l.id + order by h.id + """ + + // Binding the same pair in reverse order must preserve both relation-local schemas. + qt_latest_then_historical """ + select l.id, l.new_name, h.old_name + from ${tableName} l + join ${tableName} for version as of ${oldSnapshot} h on l.id = h.id + order by l.id + """ - // Scenario TC07-join negative contract: - // two historical relations in one statement currently reuse the first schema. - test { - sql """ + // Scenario TC07-join: each historical relation keeps its own schema. + qt_historical_join """ select o.id, o.old_name, n.new_name from ( select id, old_name @@ -107,13 +121,10 @@ suite("test_iceberg_schema_dual_relation_matrix", from ${tableName} for version as of ${newSnapshot} ) n on o.id = n.id order by o.id - """ - exception "Unknown column 'new_name'" - } + """ // Scenario TC07-reverse-join: binding must be independent of relation order. - test { - sql """ + qt_reverse_historical_join """ select n.id, n.new_name, o.old_name from ( select id, new_name @@ -124,39 +135,30 @@ suite("test_iceberg_schema_dual_relation_matrix", from ${tableName} for version as of ${oldSnapshot} ) o on n.id = o.id order by n.id - """ - exception "Unknown column 'old_name'" - } + """ // Scenario TC07-union: top-level historical schemas stay relation-local. - test { - sql """ + qt_historical_union """ select id, old_name as name_value from ${tableName} for version as of ${oldSnapshot} union all select id, new_name as name_value from ${tableName} for version as of ${newSnapshot} order by id, name_value - """ - exception "Unknown column 'new_name'" - } + """ // Scenario TC07-nested-union: nested field lookup is also relation-local. - test { - sql """ + qt_nested_historical_union """ select id, info.added as nested_value from ${tableName} for version as of ${oldSnapshot} union all select id, info.renamed as nested_value from ${tableName} for version as of ${newSnapshot} order by id, nested_value - """ - exception "No such struct field 'renamed'" - } + """ // Scenario TC07-CTE: CTE boundaries must not collapse snapshot schemas. - test { - sql """ + qt_historical_cte """ with old_ref as ( select id, old_name from ${tableName} for version as of ${oldSnapshot} @@ -167,13 +169,10 @@ suite("test_iceberg_schema_dual_relation_matrix", select old_ref.id, old_ref.old_name, new_ref.new_name from old_ref join new_ref on old_ref.id = new_ref.id order by old_ref.id - """ - exception "Unknown column 'new_name'" - } + """ // Scenario TC07-correlated-subquery: subqueries require an independent schema. - test { - sql """ + qt_historical_correlated_subquery """ select o.id, o.old_name from ${tableName} for version as of ${oldSnapshot} o where exists ( @@ -182,9 +181,7 @@ suite("test_iceberg_schema_dual_relation_matrix", where n.id = o.id and n.new_name is not null ) order by o.id - """ - exception "Unknown column 'new_name'" - } + """ } finally { sql """drop catalog if exists ${catalogName}""" } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy index 45307819e599a2..92bcbd358dc272 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy @@ -58,6 +58,8 @@ suite("test_iceberg_schema_metadata_atomicity_matrix", sql """switch ${catalogName}""" sql """create database if not exists ${dbName}""" sql """use ${dbName}""" + // DESC hides comments by default, so enable them before validating Iceberg field docs. + sql """set show_column_comment_in_describe=true""" try { sql """drop table if exists ${tableName}""" @@ -89,9 +91,8 @@ suite("test_iceberg_schema_metadata_atomicity_matrix", it == null ? "" : it.toString() }.join(" ") assertTrue(sparkDescriptionText.contains("top-level-comment")) - // Negative contract: Doris DESC currently omits Iceberg field comments. - assertFalse(descAfterComment.contains("top-level-comment")) - assertFalse(descAfterComment.contains("nested-comment")) + assertTrue(descAfterComment.contains("top-level-comment")) + assertTrue(descAfterComment.contains("nested-comment")) assertEquals(initialSnapshots, snapshotCount()) // Scenario S19-nullability: relaxing required to optional preserves data and historical refs. diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.groovy index 607895a6e21fb4..03fa86835d16a1 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.groovy @@ -88,38 +88,38 @@ suite("test_iceberg_schema_ref_actions_matrix", String rollbackNewSnapshot = snapshots(rollbackTable).last() sql """alter table ${rollbackTable} create tag rollback_new""" - assertEquals([[1, "old-1", 10]], sql(""" + qt_rollback_old_snapshot """ select id, old_name, payload.old_child from ${rollbackTable} for version as of ${rollbackOldSnapshot} order by id - """)) - assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + """ + qt_rollback_new_snapshot """ select id, new_name, payload.new_child from ${rollbackTable} for version as of ${rollbackNewSnapshot} order by id - """)) + """ sql """ alter table ${rollbackTable} execute rollback_to_snapshot('snapshot_id'='${rollbackOldSnapshot}') """ sql """refresh table ${rollbackTable}""" - assertEquals([[1, "old-1", 10]], sql(""" + qt_rollback_current_after_snapshot """ select id, new_name, payload.new_child from ${rollbackTable} order by id - """)) + """ assertUnknownColumn("""select old_name from ${rollbackTable}""", "old_name") - assertEquals([[1, "old-1", 10]], sql(""" + qt_rollback_old_tag """ select id, old_name, payload.old_child from ${rollbackTable}@tag(rollback_old) order by id - """)) - assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + """ + qt_rollback_new_tag """ select id, new_name, payload.new_child from ${rollbackTable}@tag(rollback_new) order by id - """)) + """ // Scenario T16-cherrypick: append snapshot with the renamed schema can be replayed after rollback. sql """ @@ -127,16 +127,16 @@ suite("test_iceberg_schema_ref_actions_matrix", execute cherrypick_snapshot('snapshot_id'='${rollbackNewSnapshot}') """ sql """refresh table ${rollbackTable}""" - assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + qt_cherrypick_current """ select id, new_name, payload.new_child from ${rollbackTable} order by id - """)) - assertEquals([[1, "old-1", 10]], sql(""" + """ + qt_cherrypick_old_tag """ select id, old_name, payload.old_child from ${rollbackTable}@tag(rollback_old) order by id - """)) + """ // Scenario T15-timestamp: timestamp rollback also keeps the latest current schema. String timestampTable = "rollback_timestamp_schema_timeline" @@ -169,14 +169,14 @@ suite("test_iceberg_schema_ref_actions_matrix", execute rollback_to_timestamp('timestamp'='${rollbackTimestamp}') """ sql """refresh table ${timestampTable}""" - assertEquals([[1, "old-1"]], sql(""" + qt_timestamp_current """ select id, new_name from ${timestampTable} order by id - """)) - assertEquals([[1, "old-1"]], sql(""" + """ + qt_timestamp_old_snapshot """ select id, old_name from ${timestampTable} for version as of ${timestampOldSnapshot} order by id - """)) + """ // Scenario T16-fast-forward: advance a pre-rename branch to the renamed main schema. String fastForwardTable = "fast_forward_schema_timeline" @@ -190,28 +190,26 @@ suite("test_iceberg_schema_ref_actions_matrix", properties ('format-version'='2', 'write.format.default'='parquet') """ sql """insert into ${fastForwardTable} values (1, 'old-1', 10)""" + String preRenameSnapshot = snapshots(fastForwardTable).last() sql """alter table ${fastForwardTable} create branch pre_rename_branch""" sql """alter table ${fastForwardTable} create tag pre_rename_tag""" sql """alter table ${fastForwardTable} rename column old_name new_name""" sql """alter table ${fastForwardTable} modify column metric bigint""" sql """insert into ${fastForwardTable} values (2, 'new-2', 6000000000)""" - // Scenario T08 negative contract: before fast-forward, branch reads use the latest rename schema. - test { - sql """ - select id, old_name, metric + // Scenario T08: before fast-forward, the branch keeps old data under the current table schema. + qt_pre_fast_forward_branch """ + select id, new_name, metric from ${fastForwardTable}@branch(pre_rename_branch) order by id - """ - exception "Unknown column 'old_name'" - } - assertEquals([[1, "old-1", 10]], sql(""" + """ + qt_pre_fast_forward_tag """ select id, old_name, metric from ${fastForwardTable}@tag(pre_rename_tag) order by id - """)) + """ - // Scenario T09 negative contract: a pre-rename branch write uses main's latest schema. + // Scenario T09: writes use the table's latest schema even when targeting an old branch. test { sql """ insert into ${fastForwardTable}@branch(pre_rename_branch) @@ -220,21 +218,29 @@ suite("test_iceberg_schema_ref_actions_matrix", exception "Unknown column 'old_name'" } + // A historical source relation must not replace the latest schema used to bind the branch target. + sql """ + explain insert into ${fastForwardTable}@branch(pre_rename_branch) + (id, new_name, metric) + select id, old_name, metric + from ${fastForwardTable} for version as of ${preRenameSnapshot} + """ + sql """ alter table ${fastForwardTable} execute fast_forward('branch'='pre_rename_branch', 'to'='main') """ sql """refresh table ${fastForwardTable}""" - assertEquals([[1, "old-1", 10L], [2, "new-2", 6000000000L]], sql(""" + qt_post_fast_forward_branch """ select id, new_name, metric from ${fastForwardTable}@branch(pre_rename_branch) order by id - """)) - assertEquals([[1, "old-1", 10]], sql(""" + """ + qt_post_fast_forward_tag """ select id, old_name, metric from ${fastForwardTable}@tag(pre_rename_tag) order by id - """)) + """ } finally { sql """drop database if exists ${dbName} force""" sql """drop catalog if exists ${catalogName}""" diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy index e0876e17ea5833..f92f3b83c59c70 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy @@ -424,7 +424,7 @@ suite("test_iceberg_schema_time_travel_matrix", limit 1 """)[0][0].toString() - // Scenario TC02/T01/T05/T06/T08: every old reference exposes its own nested schema. + // Scenario TC02/T01/T05/T06: snapshots and tags expose their historical nested schema. assertEquals([[1, 10, 100, 1000]], sql(""" select id, info.metric, events[1].score, attrs['k'].code @@ -443,13 +443,14 @@ suite("test_iceberg_schema_time_travel_matrix", from ${dorisNestedTable}@tag(doris_nested_cp0) order by id """)) - // Negative contract: an old branch currently leaks the latest BIGINT nested types. - assertEquals([[1, 10L, 100L, 1000L]], - sql(""" - select id, info.metric, events[1].score, attrs['k'].code - from ${dorisNestedTable}@branch(doris_nested_cp0_branch) - order by id - """)) + // Named branches keep their head rows but expose renamed fields from the current schema. + qt_doris_nested_branch_current_schema """ + select id, info.metric, info.renamed, + events[1].score, events[1].renamed, + attrs['k'].code, attrs['k'].renamed + from ${dorisNestedTable}@branch(doris_nested_cp0_branch) + order by id + """ assertUnknownColumn(""" select info.renamed from ${dorisNestedTable} for version as of ${dorisNestedCp0} @@ -530,7 +531,7 @@ suite("test_iceberg_schema_time_travel_matrix", sql("""select id, metric from ${topTable} where metric > 5000000000 order by id""")) assertUnknownColumn("""select old_name from ${topTable}""", "old_name") - // Scenario T01/T05/T06/T08: the pre-change snapshot/tag/branch binds old_name and victim. + // Scenario T01/T05/T06: the pre-change snapshot and tag bind old_name and old victim. List> topCp0Rows = [[1, "alpha", "old-v1", 10]] assertEquals(topCp0Rows, sql(""" select id, old_name, victim, metric @@ -547,15 +548,11 @@ suite("test_iceberg_schema_time_travel_matrix", from ${topTable}@tag(top_cp0) order by id """)) - // Negative contract: an old branch is currently analyzed with the latest rename schema. - test { - sql """ - select id, old_name, victim, metric - from ${topTable}@branch(top_cp0_branch) - order by id - """ - exception "Unknown column 'old_name'" - } + qt_top_branch_current_schema """ + select id, MixedName, victim, metric + from ${topTable}@branch(top_cp0_branch) + order by id + """ assertUnknownColumn(""" select MixedName from ${topTable} for version as of ${topCp0} """, "MixedName") diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.groovy index f0f795477396ef..722185244ff3da 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.groovy @@ -92,15 +92,12 @@ suite("test_paimon_schema_branch_partition_matrix", "p0,external,paimon") { from ${branchTable} order by id """)) - // Negative contract: Doris cannot initialize a Paimon branch with an independent schema. - test { - sql """ - select id, branch_name, metric, branch_only - from ${branchTable}@branch(schema_branch) - order by id - """ - exception "failed to initSchema" - } + // The branch schema is independent from main and must be loaded from the branch table. + qt_independent_branch_schema """ + select id, branch_name, metric, branch_only + from ${branchTable}@branch(schema_branch) + order by id + """ test { sql """select branch_only from ${branchTable}""" exception "Unknown column 'branch_only'" diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.groovy index b5478dfbea7348..06c2e0f9ea985d 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.groovy @@ -79,21 +79,19 @@ suite("test_paimon_schema_dual_relation_matrix", "p0,external,paimon") { sql """use ${dbName}""" // Scenario TC07-baseline: verify a single historical relation binds its own schema. - assertEquals([[1, "old-1", 10]], sql(""" + qt_old_snapshot """ select id, old_name, info.added from ${tableName} for version as of ${oldSnapshot} order by id - """)) - assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + """ + qt_new_snapshot """ select id, new_name, info.renamed from ${tableName} for version as of ${newSnapshot} order by id - """)) + """ - // Scenario TC07-join negative contract: - // two Paimon historical relations currently reuse the first schema. - test { - sql """ + // Scenario TC07-join: each historical relation keeps its own schema. + qt_historical_join """ select o.id, o.old_name, n.new_name from ( select id, old_name @@ -104,13 +102,10 @@ suite("test_paimon_schema_dual_relation_matrix", "p0,external,paimon") { from ${tableName} for version as of ${newSnapshot} ) n on o.id = n.id order by o.id - """ - exception "Unknown column 'new_name'" - } + """ // Scenario TC07-reverse-join: binding must be independent of relation order. - test { - sql """ + qt_reverse_historical_join """ select n.id, n.new_name, o.old_name from ( select id, new_name @@ -121,39 +116,30 @@ suite("test_paimon_schema_dual_relation_matrix", "p0,external,paimon") { from ${tableName} for version as of ${oldSnapshot} ) o on n.id = o.id order by n.id - """ - exception "Unknown column 'old_name'" - } + """ // Scenario TC07-union: top-level historical schemas stay relation-local. - test { - sql """ + qt_historical_union """ select id, old_name as name_value from ${tableName} for version as of ${oldSnapshot} union all select id, new_name as name_value from ${tableName} for version as of ${newSnapshot} order by id, name_value - """ - exception "Unknown column 'new_name'" - } + """ // Scenario TC07-nested-union: nested lookup is also relation-local. - test { - sql """ + qt_nested_historical_union """ select id, info.added as nested_value from ${tableName} for version as of ${oldSnapshot} union all select id, info.renamed as nested_value from ${tableName} for version as of ${newSnapshot} order by id, nested_value - """ - exception "No such struct field 'renamed'" - } + """ // Scenario TC07-CTE: CTE boundaries must not collapse snapshot schemas. - test { - sql """ + qt_historical_cte """ with old_ref as ( select id, old_name from ${tableName} for version as of ${oldSnapshot} @@ -164,13 +150,10 @@ suite("test_paimon_schema_dual_relation_matrix", "p0,external,paimon") { select old_ref.id, old_ref.old_name, new_ref.new_name from old_ref join new_ref on old_ref.id = new_ref.id order by old_ref.id - """ - exception "Unknown column 'new_name'" - } + """ // Scenario TC07-correlated-subquery: subqueries require an independent schema. - test { - sql """ + qt_historical_correlated_subquery """ select o.id, o.old_name from ${tableName} for version as of ${oldSnapshot} o where exists ( @@ -179,9 +162,7 @@ suite("test_paimon_schema_dual_relation_matrix", "p0,external,paimon") { where n.id = o.id and n.new_name is not null ) order by o.id - """ - exception "Unknown column 'new_name'" - } + """ } finally { sql """drop catalog if exists ${catalogName}""" } diff --git a/regression-test/suites/schema_change_p0/test_agg_schema_value_drop.groovy b/regression-test/suites/schema_change_p0/test_agg_schema_value_drop.groovy index 7f5c77192f5f36..544a24cd28c5c2 100644 --- a/regression-test/suites/schema_change_p0/test_agg_schema_value_drop.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_schema_value_drop.groovy @@ -1053,8 +1053,9 @@ suite("test_agg_schema_value_drop", "p0") { // Test the AGGREGATE model by drop a value type from STRUCT - errorMessage = "errCode = 2, detailMessage = can not cast from origin type STRUCT to target type=ARRAY" - expectException({ + // Field nullability is inferred independently; keep this assertion focused on the incompatible shapes. + errorMessage = "errCode = 2, detailMessage = can not cast from origin type STRUCT<" + expectExceptionLike({ sql initTable sql initTableData sql """ alter table ${tbName1} DROP column s_info """ @@ -1063,7 +1064,7 @@ suite("test_agg_schema_value_drop", "p0") { sql getTableStatusSql time 600 }, insertSql, true, "${tbName1}") - }, errorMessage) + }, errorMessage, "to target type=ARRAY") } diff --git a/regression-test/suites/schema_change_p0/test_dup_schema_value_drop.groovy b/regression-test/suites/schema_change_p0/test_dup_schema_value_drop.groovy index 842f26929fe37c..31f69762c0c189 100644 --- a/regression-test/suites/schema_change_p0/test_dup_schema_value_drop.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_schema_value_drop.groovy @@ -1053,8 +1053,9 @@ suite("test_dup_schema_value_drop", "p0") { // Test the duplicate model by drop a value type from STRUCT - errorMessage = "errCode = 2, detailMessage = can not cast from origin type STRUCT to target type=ARRAY" - expectException({ + // Field nullability is inferred independently; keep this assertion focused on the incompatible shapes. + errorMessage = "errCode = 2, detailMessage = can not cast from origin type STRUCT<" + expectExceptionLike({ sql initTable sql initTableData sql """ alter table ${tbName1} DROP column s_info """ @@ -1063,7 +1064,7 @@ suite("test_dup_schema_value_drop", "p0") { sql getTableStatusSql time 600 }, insertSql, true, "${tbName1}") - }, errorMessage) + }, errorMessage, "to target type=ARRAY") }