diff --git a/src/Storages/MergeTree/ExportPartTask.cpp b/src/Storages/MergeTree/ExportPartTask.cpp index 2d1ac9155dbff..644a8073d8bd5 100644 --- a/src/Storages/MergeTree/ExportPartTask.cpp +++ b/src/Storages/MergeTree/ExportPartTask.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "Common/setThreadName.h" #include @@ -384,6 +385,16 @@ bool ExportPartTask::executeStep() auto pipeline_settings = BuildQueryPipelineSettings(local_context); auto builder = plan_for_part.buildQueryPipeline(optimization_settings, pipeline_settings); + /// The sink of a storage without sparse serialization, and the output format it writes with, + /// expect full columns, as `INSERT` guarantees for it. + if (!destination_storage->supportsSparseSerialization()) + { + builder->addSimpleTransform([](const SharedHeader & header) -> ProcessorPtr + { + return std::make_shared(header); + }); + } + builder->setProgressCallback([&exports_list_entry](const Progress & progress) { (*exports_list_entry)->bytes_read_uncompressed += progress.read_bytes; diff --git a/tests/queries/0_stateless/05386_export_merge_tree_part_sparse_columns.reference b/tests/queries/0_stateless/05386_export_merge_tree_part_sparse_columns.reference new file mode 100644 index 0000000000000..7f0c8163bc0c7 --- /dev/null +++ b/tests/queries/0_stateless/05386_export_merge_tree_part_sparse_columns.reference @@ -0,0 +1,7 @@ +---- Sparse column cast to a wider type, first granule without zeros +Sparse +300000 0 +---- Sparse Nullable column, null counts +Sparse +299699 0 +299699 diff --git a/tests/queries/0_stateless/05386_export_merge_tree_part_sparse_columns.sh b/tests/queries/0_stateless/05386_export_merge_tree_part_sparse_columns.sh new file mode 100755 index 0000000000000..1c2b9fd870fc6 --- /dev/null +++ b/tests/queries/0_stateless/05386_export_merge_tree_part_sparse_columns.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: requires Iceberg + +# EXPORT PART of a part with sparse columns into an Iceberg table: +# - cast to a wider type, the granule without zeros becomes a full chunk and the others stay sparse, +# which the Parquet writer must not concatenate into one row group as they are; +# - the null counts written to the manifest must count the NULLs of a sparse `Nullable` column. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# shellcheck source=./export_part.lib +. "$CUR_DIR"/export_part.lib + +src="src_${RANDOM}" +dst="dst_${RANDOM}" +src_nullable="src_nullable_${RANDOM}" +dst_nullable="dst_nullable_${RANDOM}" +dst_path="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}_${dst}/" +dst_nullable_path="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}_${dst_nullable}/" + +query() { + $CLICKHOUSE_CLIENT --query "$1" +} + +source_settings="ratio_of_defaults_for_sparse_serialization = 0.9, serialization_info_version = 'with_types', min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, index_granularity = 8192, index_granularity_bytes = '10Mi'" +insert_settings="max_insert_threads = 1, max_block_size = 300000, max_insert_block_size = 300000, min_insert_block_size_rows = 300000, min_insert_block_size_bytes = 0" +# One row group holds every chunk of the part. +export_settings="allow_experimental_export_merge_tree_part = 1, allow_insert_into_iceberg = 1, output_format_parquet_row_group_size = 1000000, output_format_parquet_row_group_size_bytes = 536870912" + +echo "---- Sparse column cast to a wider type, first granule without zeros" +query "CREATE TABLE $src (id Int64, f UInt8) ENGINE = MergeTree ORDER BY id SETTINGS $source_settings, nullable_serialization_version = 'basic'" +query "INSERT INTO $src SELECT number, number < 10000 OR number % 997 = 0 FROM numbers(300000) SETTINGS $insert_settings" +query "SELECT serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = '$src' AND column = 'f' AND active" +query "CREATE TABLE $dst (id Int64, f Int32) ENGINE = IcebergLocal('$dst_path', 'Parquet')" + +part=$(query "SELECT name FROM system.parts WHERE database = currentDatabase() AND table = '$src' AND active") +query "ALTER TABLE $src EXPORT PART '$part' TO TABLE $dst SETTINGS $export_settings" +wait_for_exports 1 + +query "SELECT count(), countIf(f != (id < 10000 OR id % 997 = 0)) FROM $dst" + +echo "---- Sparse Nullable column, null counts" +query "CREATE TABLE $src_nullable (id Int64, n Nullable(Int32)) ENGINE = MergeTree ORDER BY id SETTINGS $source_settings, nullable_serialization_version = 'allow_sparse'" +query "INSERT INTO $src_nullable SELECT number, if(number % 997 = 0, toInt32(number), NULL) FROM numbers(300000) SETTINGS $insert_settings" +query "SELECT serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = '$src_nullable' AND column = 'n' AND active" +query "CREATE TABLE $dst_nullable (id Int64, n Nullable(Int32)) ENGINE = IcebergLocal('$dst_nullable_path', 'Parquet')" + +part=$(query "SELECT name FROM system.parts WHERE database = currentDatabase() AND table = '$src_nullable' AND active") +query "ALTER TABLE $src_nullable EXPORT PART '$part' TO TABLE $dst_nullable SETTINGS $export_settings" +wait_for_exports 2 + +query "SELECT countIf(n IS NULL), countIf(isNull(n) != (id % 997 != 0)) FROM $dst_nullable" +# Files are pruned by the null counts of the manifest. +query "SELECT count() FROM $dst_nullable WHERE n IS NULL SETTINGS use_iceberg_partition_pruning = 1" + +query "DROP TABLE $src, $dst, $src_nullable, $dst_nullable" +rm -rf "$dst_path" "$dst_nullable_path"