From 99f49875e1f17bf2707eb5ace2d26961e931d7eb Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Tue, 28 Jul 2026 11:01:20 +0800 Subject: [PATCH] feat(blob): support placeholder fallback for partial updates A data-evolution partial update rewrites only the touched rows of a blob column and records every untouched row as a placeholder entry (bin_length -2, no data bytes). The placeholder protocol is confined to two internal channels. On write, only a data-evolution blob-only column write enables it (BlobDefs::kWritePlaceholderKey); under it a value exactly equal to the internal reserved sentinel _PAIMON_BLOB_PLACEHOLDER is persisted as a bin_length -2 entry. Every other write stores blob bytes verbatim. On read, placeholder entries are an error unless BlobDefs::kEmitPlaceholderSentinelKey switches the reader to emit the sentinel for them. Placeholders are identified by exact byte equality with the sentinel, so a user blob whose bytes equal it collides with the marker: written through the partial-update channel it is persisted as a placeholder entry that a single-layer read rejects loudly, and left untouched in an older layer under a later partial update it reads as a placeholder in every layer and silently degrades to a null blob. These collisions are accepted as negligibly improbable and pinned by tests. The blob.internal.* keys are stripped from user-supplied table options on both the write and read paths, so only the internal channels can enable the protocol. BlobBunch keeps all max-sequence layers of a bunch, and the new BlobFallbackBatchReader resolves each row to the newest layer holding a real value: an explicitly written null wins over older layers, a row that is a placeholder in every layer degrades to a null blob while keeping its _ROW_ID and reporting -1 as its _SEQUENCE_NUMBER, resolved rows report their layer's sequence number, and row-range pushdown is honored including the row id ranges a layer does not cover. BlobFileBatchReader::GetPreviousBatchFileRowId now maps batch positions back to original file row indexes through target_blob_row_indexes_, so _ROW_ID completion keeps working when a selection bitmap removed rows. CompleteRowTrackingFieldsBatchReader takes the physical field names of a format without a self-describing schema (blob) from DataFileMeta::write_cols instead of treating NotImplemented as an empty file schema. DataEvolutionFileReader::Create accepts a single inner reader: a blob-only write collapses every layer of a merge split into one blob bunch, and the degenerate union still maps the bunch's fields into read schema order and null fills unmatched read fields. Reader offsets are now validated against the reader count instead of requiring at least two readers. DataEvolutionBatchScan::WrapToIndexedSplits computed a split's row id range from its first and last file only, so a partial-update blob layer covering a strict subrange of the split truncated the range and silently dropped requested row ranges at scan time. Because the row-id ranges of the files in a split may be unordered, discontiguous, or overlapping, the index is now intersected with each file's row-id range separately and the results are sorted and merged, aligning with Java's DataEvolutionBatchScan.wrap(). Co-Authored-By: Claude Fable 5 --- src/paimon/CMakeLists.txt | 3 + src/paimon/common/data/blob_defs.h | 62 ++ .../reader/blob_fallback_batch_reader.cpp | 403 ++++++++++++ .../reader/blob_fallback_batch_reader.h | 159 +++++ .../blob_fallback_batch_reader_test.cpp | 358 +++++++++++ .../reader/data_evolution_file_reader.cpp | 9 +- .../reader/data_evolution_file_reader.h | 6 +- .../data_evolution_file_reader_test.cpp | 16 +- src/paimon/core/append/append_only_writer.cpp | 16 +- src/paimon/core/append/append_only_writer.h | 2 +- .../core/io/blob_data_file_writer_factory.cpp | 17 +- .../core/io/blob_data_file_writer_factory.h | 6 +- .../complete_row_tracking_fields_reader.cpp | 28 +- .../io/complete_row_tracking_fields_reader.h | 12 +- ...mplete_row_tracking_fields_reader_test.cpp | 48 +- src/paimon/core/mergetree/lookup_levels.cpp | 3 +- .../core/operation/abstract_split_read.cpp | 27 +- .../core/operation/abstract_split_read.h | 10 +- .../operation/data_evolution_split_read.cpp | 234 +++++-- .../operation/data_evolution_split_read.h | 46 +- .../data_evolution_split_read_test.cpp | 141 +++-- .../core/operation/merge_file_split_read.cpp | 6 +- .../core/operation/raw_file_split_read.cpp | 3 +- .../source/data_evolution_batch_scan.cpp | 25 +- .../table/source/data_evolution_batch_scan.h | 11 +- .../source/data_evolution_batch_scan_test.cpp | 113 ++++ .../format/blob/blob_file_batch_reader.cpp | 50 +- .../format/blob/blob_file_batch_reader.h | 57 +- .../blob/blob_file_batch_reader_test.cpp | 98 ++- src/paimon/format/blob/blob_format_writer.cpp | 23 +- src/paimon/format/blob/blob_format_writer.h | 11 +- .../format/blob/blob_format_writer_test.cpp | 342 ++++++++-- src/paimon/format/blob/blob_reader_builder.h | 6 +- .../format/blob/blob_stats_extractor.cpp | 5 +- src/paimon/format/blob/blob_writer_builder.h | 6 +- test/inte/blob_table_inte_test.cpp | 590 ++++++++++++++++++ 36 files changed, 2672 insertions(+), 280 deletions(-) create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader.cpp create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader.h create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader_test.cpp create mode 100644 src/paimon/core/table/source/data_evolution_batch_scan_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 07b033734..3b97dfd68 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -131,6 +131,7 @@ set(PAIMON_COMMON_SRCS common/reader/predicate_batch_reader.cpp common/reader/prefetch_file_batch_reader_impl.cpp common/reader/reader_utils.cpp + common/reader/blob_fallback_batch_reader.cpp common/reader/blob_view_resolving_batch_reader.cpp common/reader/complete_row_kind_batch_reader.cpp common/reader/data_evolution_file_reader.cpp @@ -542,6 +543,7 @@ if(PAIMON_BUILD_TESTS) common/reader/prefetch_file_batch_reader_impl_test.cpp common/reader/reader_utils_test.cpp common/reader/complete_row_kind_batch_reader_test.cpp + common/reader/blob_fallback_batch_reader_test.cpp common/reader/blob_view_resolving_batch_reader_test.cpp common/reader/data_evolution_file_reader_test.cpp common/reader/data_evolution_array_test.cpp @@ -803,6 +805,7 @@ if(PAIMON_BUILD_TESTS) core/table/source/table_read_test.cpp core/table/source/append_count_reader_test.cpp core/table/source/pk_count_reader_test.cpp + core/table/source/data_evolution_batch_scan_test.cpp core/table/source/data_split_test.cpp core/table/source/deletion_file_test.cpp core/table/source/split_generator_test.cpp diff --git a/src/paimon/common/data/blob_defs.h b/src/paimon/common/data/blob_defs.h index b8bb6f308..cd5d24323 100644 --- a/src/paimon/common/data/blob_defs.h +++ b/src/paimon/common/data/blob_defs.h @@ -17,6 +17,10 @@ #pragma once #include +#include +#include +#include +#include namespace paimon { @@ -45,6 +49,64 @@ class BlobDefs { /// A bin_length value of -1 in the index indicates a null blob entry. static constexpr int64_t kNullBinLength = -1; + /// A bin_length value of -2 in the index indicates a placeholder blob entry, written by + /// data-evolution partial updates for rows whose blob value is not updated. A placeholder + /// entry occupies no file space; readers must fall back to an older blob file covering the + /// same row to resolve the value. Aligned with Java's BlobFormatWriter.PLACE_HOLDER_LENGTH. + static constexpr int64_t kPlaceholderBinLength = -2; + /// Sentinel bytes standing for a placeholder blob value in two internal channels: + /// + /// - Write channel: a data-evolution partial update (a blob-only column write, see + /// kWritePlaceholderKey) marks a not-updated row with these bytes, and the blob format + /// writer persists it as a bin_length -2 entry. Use PlaceholderSentinelView() to build + /// such write arrays. Outside that mode the writer never interprets values, so arbitrary + /// user bytes can never be turned into a placeholder entry. + /// - Read channel: a placeholder-aware reader (see kEmitPlaceholderSentinelKey) emits these + /// bytes for -2 entries so the fallback merge can identify placeholders after the batch + /// has passed through schema-mapping readers. + /// + /// Both channels identify a placeholder by exact byte equality with this internal reserved + /// value (IsPlaceholderSentinel), and the fallback merge byte-compares every layer of a + /// bunch — including files written outside the write channel. A user blob whose bytes + /// exactly equal the marker therefore collides with it in two ways: written through the + /// partial-update channel it is persisted as a placeholder entry, which a single-layer read + /// rejects loudly (no older layer can resolve it); left untouched in an older layer under a + /// later partial update it reads as a placeholder in every layer and silently degrades to a + /// null blob. The marker is distinctive enough that these collisions are accepted as + /// negligibly improbable. Sentinel bytes are never stored in blob files. + static constexpr char kPlaceholderSentinel[] = "_PAIMON_BLOB_PLACEHOLDER"; + /// Byte length of kPlaceholderSentinel, excluding the literal's terminating NUL. + static constexpr int32_t kPlaceholderSentinelLength = sizeof(kPlaceholderSentinel) - 1; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// reader emits kPlaceholderSentinel for placeholder entries instead of failing on them. + /// Only the data-evolution blob fallback read path sets this. + static constexpr char kEmitPlaceholderSentinelKey[] = "blob.internal.emit-placeholder-sentinel"; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// format writer persists a value exactly equal to kPlaceholderSentinel as a bin_length -2 + /// entry. Only set for data-evolution partial updates, i.e. blob-only column writes of a + /// table with data evolution enabled; all other writes store bytes verbatim. + static constexpr char kWritePlaceholderKey[] = "blob.internal.write-placeholder"; + + /// The sentinel bytes for building a data-evolution partial-update write array: a row equal + /// to this view is persisted as a placeholder entry (see kWritePlaceholderKey). + static std::string_view PlaceholderSentinelView() { + return {kPlaceholderSentinel, static_cast(kPlaceholderSentinelLength)}; + } + + /// True when the bytes are exactly the placeholder sentinel. + static bool IsPlaceholderSentinel(const char* data, size_t size) { + return size == static_cast(kPlaceholderSentinelLength) && + memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0; + } + + /// Removes the internal placeholder option keys from a format options map. The placeholder + /// channels must only ever be enabled by the internal data-evolution write and read paths, + /// so every consumer building format options from user-supplied table options strips these + /// keys before applying its own decision. + static void EraseInternalPlaceholderOptions(std::map* options) { + options->erase(kEmitPlaceholderSentinelKey); + options->erase(kWritePlaceholderKey); + } /// Blob file format version. static constexpr int8_t kFileVersion = 1; /// Magic number identifying the start of each blob bin. diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp new file mode 100644 index 000000000..0d5a6c72d --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp @@ -0,0 +1,403 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/reader/reader_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> BlobFallbackBatchReader::Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool) { + if (sequence_groups.size() < 2) { + return Status::Invalid( + "Blob fallback needs at least two sequence groups; a single group should be read " + "sequentially."); + } + if (read_schema == nullptr) { + return Status::Invalid("Blob fallback read schema cannot be nullptr."); + } + if (read_batch_size <= 0) { + return Status::Invalid(fmt::format( + "Blob fallback read batch size '{}' should be larger than zero", read_batch_size)); + } + int32_t blob_field_idx = -1; + for (int32_t i = 0; i < read_schema->num_fields(); i++) { + if (BlobUtils::IsBlobField(read_schema->field(i))) { + if (blob_field_idx != -1) { + return Status::Invalid( + "Blob fallback read schema should contain exactly one blob field."); + } + blob_field_idx = i; + } + } + if (blob_field_idx == -1) { + return Status::Invalid("Blob fallback read schema should contain a blob field."); + } + int32_t row_id_field_idx = read_schema->GetFieldIndex(SpecialFields::RowId().Name()); + int32_t seq_num_field_idx = read_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); + std::vector groups; + groups.reserve(sequence_groups.size()); + for (auto& segments : sequence_groups) { + if (segments.empty()) { + return Status::Invalid("Blob fallback sequence group should not be empty."); + } + for (const auto& segment : segments) { + if (segment.reader == nullptr && segment.gap_selected_ranges.empty()) { + return Status::Invalid( + "Blob fallback gap segment should cover at least one selected row id."); + } + } + GroupCursor cursor; + cursor.segments = std::move(segments); + groups.push_back(std::move(cursor)); + } + return std::unique_ptr( + new BlobFallbackBatchReader(std::move(groups), read_schema, blob_field_idx, + row_id_field_idx, seq_num_field_idx, read_batch_size, pool)); +} + +BlobFallbackBatchReader::BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool) + : groups_(std::move(groups)), + read_schema_(read_schema), + blob_field_idx_(blob_field_idx), + row_id_field_idx_(row_id_field_idx), + seq_num_field_idx_(seq_num_field_idx), + read_batch_size_(read_batch_size), + arrow_pool_(GetArrowPool(pool)) {} + +Result BlobFallbackBatchReader::FillWindow(size_t group_idx, int64_t want, + std::vector* chunks) { + GroupCursor& cursor = groups_[group_idx]; + int64_t collected = 0; + while (collected < want) { + if (!cursor.pending.empty()) { + const std::shared_ptr& front = cursor.pending.front(); + int64_t available = front->length() - cursor.pending_pos; + int64_t take = std::min(available, want - collected); + chunks->push_back(Chunk{front, cursor.pending_pos, take, {}}); + cursor.pending_pos += take; + collected += take; + if (cursor.pending_pos == front->length()) { + cursor.pending.pop_front(); + cursor.pending_pos = 0; + } + continue; + } + if (cursor.segment_idx >= cursor.segments.size()) { + // group exhausted; only the first group may define a shorter window + break; + } + Segment& segment = cursor.segments[cursor.segment_idx]; + if (segment.reader == nullptr) { + // gap segment: all rows are placeholders, stepped range by range so the row ids + // stay available for all-placeholder rows + if (cursor.gap_range_idx >= segment.gap_selected_ranges.size()) { + cursor.segment_idx++; + cursor.gap_range_idx = 0; + cursor.gap_range_pos = 0; + continue; + } + const Range& range = segment.gap_selected_ranges[cursor.gap_range_idx]; + int64_t remaining = range.Count() - cursor.gap_range_pos; + int64_t take = std::min(remaining, want - collected); + Chunk chunk{nullptr, 0, take, {}}; + if (row_id_field_idx_ >= 0) { + chunk.gap_row_ids.reserve(take); + for (int64_t k = 0; k < take; k++) { + chunk.gap_row_ids.push_back(range.from + cursor.gap_range_pos + k); + } + } + chunks->push_back(std::move(chunk)); + cursor.gap_range_pos += take; + collected += take; + if (cursor.gap_range_pos == range.Count()) { + cursor.gap_range_idx++; + cursor.gap_range_pos = 0; + } + continue; + } + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap, + segment.reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + cursor.segment_idx++; + continue; + } + auto& [read_batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = read_batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector selected_array_vec, + ReaderUtils::GenerateFilteredArrayVector(src_array, bitmap)); + for (const auto& selected_array : selected_array_vec) { + if (selected_array->length() == 0) { + continue; + } + auto struct_array = std::dynamic_pointer_cast(selected_array); + if (struct_array == nullptr) { + return Status::Invalid("Blob fallback expects file readers to emit struct arrays."); + } + cursor.pending.push_back(std::move(struct_array)); + } + } + return collected; +} + +Result> BlobFallbackBatchReader::ComputePlaceholderFlags( + const std::vector& chunks, int64_t row_count) const { + std::vector flags(row_count, false); + int64_t pos = 0; + for (const auto& chunk : chunks) { + if (chunk.array == nullptr) { + // gap rows stand for placeholders + std::fill(flags.begin() + pos, flags.begin() + pos + chunk.length, true); + } else { + std::shared_ptr blob_col = chunk.array->field(blob_field_idx_); + auto binary_col = std::dynamic_pointer_cast(blob_col); + if (binary_col == nullptr) { + return Status::Invalid(fmt::format( + "Blob fallback expects the blob column to be large binary, but got {}", + blob_col->type()->ToString())); + } + for (int64_t k = 0; k < chunk.length; k++) { + int64_t idx = chunk.offset + k; + if (binary_col->IsNull(idx)) { + continue; + } + std::string_view value = binary_col->GetView(idx); + if (BlobDefs::IsPlaceholderSentinel(value.data(), value.size())) { + flags[pos + k] = true; + } + } + } + pos += chunk.length; + } + return flags; +} + +Result> BlobFallbackBatchReader::AssembleRowIdRun( + const std::vector& chunks, int64_t run_start, int64_t run_end) const { + arrow::ArrayVector pieces; + int64_t pos = 0; + for (const auto& chunk : chunks) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array != nullptr) { + std::shared_ptr column = chunk.array->field(row_id_field_idx_); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } else { + arrow::Int64Builder builder(arrow_pool_.get()); + for (int64_t r = overlap_start; r < overlap_end; r++) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(chunk.gap_row_ids[r - pos])); + } + std::shared_ptr piece; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&piece)); + pieces.push_back(std::move(piece)); + } + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the + // zero-offset BatchReader contract. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result> BlobFallbackBatchReader::AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const { + const auto row_count = static_cast(group_choice.size()); + arrow::ArrayVector pieces; + int64_t run_start = 0; + while (run_start < row_count) { + const int32_t group = group_choice[run_start]; + int64_t run_end = run_start + 1; + while (run_end < row_count && group_choice[run_end] == group) { + run_end++; + } + if (group < 0) { + // placeholder in every layer: the blob degrades to null; the row keeps its row id + // (taken from the newest group, which steps in lockstep), reports -1 as its + // sequence number, and returns null for every other field + if (field_idx == row_id_field_idx_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_id_piece, + AssembleRowIdRun(group_chunks[0], run_start, run_end)); + pieces.push_back(std::move(row_id_piece)); + } else if (field_idx == seq_num_field_idx_) { + arrow::Int64Scalar seq_scalar(-1); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr seq_piece, + arrow::MakeArrayFromScalar(seq_scalar, run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(seq_piece)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_piece, + arrow::MakeArrayOfNull(read_schema_->field(field_idx)->type(), + run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(null_piece)); + } + } else { + int64_t pos = 0; + for (const auto& chunk : group_chunks[group]) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array == nullptr) { + return Status::Invalid( + "Unexpected: a gap row was chosen as a blob fallback result."); + } + std::shared_ptr column = chunk.array->field(field_idx); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + } + run_start = run_end; + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the + // zero-offset BatchReader contract. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result BlobFallbackBatchReader::NextBatch() { + if (closed_) { + return Status::Invalid("blob fallback batch reader is closed"); + } + std::vector> group_chunks(groups_.size()); + // the first (newest) group defines the window; the others must step in lockstep + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, FillWindow(0, read_batch_size_, &group_chunks[0])); + for (size_t g = 1; g < groups_.size(); g++) { + // ask for one row even when the first group is exhausted, so that a longer group is + // reported as a misalignment instead of silently truncating the read + const int64_t want = std::max(row_count, 1); + PAIMON_ASSIGN_OR_RAISE(int64_t got, FillWindow(g, want, &group_chunks[g])); + if (got != row_count) { + return Status::Invalid(fmt::format( + "All sequence groups of a blob fallback read should have the same number of " + "rows: group {} yielded {} rows in a window of {}", + g, got, row_count)); + } + } + if (row_count == 0) { + return BatchReader::MakeEofBatch(); + } + + std::vector> placeholder_flags(groups_.size()); + for (size_t g = 0; g < groups_.size(); g++) { + PAIMON_ASSIGN_OR_RAISE(placeholder_flags[g], + ComputePlaceholderFlags(group_chunks[g], row_count)); + } + // per row, the first group in max-sequence order with a real entry wins; -1 means the row is + // a placeholder in every group + std::vector group_choice(row_count, -1); + for (int64_t r = 0; r < row_count; r++) { + for (size_t g = 0; g < groups_.size(); g++) { + if (!placeholder_flags[g][r]) { + group_choice[r] = static_cast(g); + break; + } + } + } + + arrow::ArrayVector columns; + columns.reserve(read_schema_->num_fields()); + for (int32_t field_idx = 0; field_idx < read_schema_->num_fields(); field_idx++) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr column, + AssembleColumn(field_idx, group_choice, group_chunks)); + columns.push_back(std::move(column)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, + arrow::StructArray::Make(columns, read_schema_->fields())); + std::unique_ptr c_array = std::make_unique(); + std::unique_ptr c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*target_array, c_array.get(), c_schema.get())); + return std::make_pair(std::move(c_array), std::move(c_schema)); +} + +Result BlobFallbackBatchReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return BatchReader::MakeEofBatchWithBitmap(); + } + return ReaderUtils::AddAllValidBitmap(std::move(batch)); +} + +void BlobFallbackBatchReader::Close() { + for (auto& group : groups_) { + group.pending.clear(); + for (auto& segment : group.segments) { + if (segment.reader) { + segment.reader->Close(); + } + } + } + closed_ = true; +} + +std::shared_ptr BlobFallbackBatchReader::GetReaderMetrics() const { + auto metrics = std::make_shared(); + for (const auto& group : groups_) { + for (const auto& segment : group.segments) { + if (segment.reader) { + auto reader_metrics = segment.reader->GetReaderMetrics(); + if (reader_metrics) { + metrics->Merge(reader_metrics); + } + } + } + } + return metrics; +} + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.h b/src/paimon/common/reader/blob_fallback_batch_reader.h new file mode 100644 index 000000000..471054610 --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.h @@ -0,0 +1,159 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/utils/range.h" + +namespace arrow { +class Array; +class StructArray; +} // namespace arrow + +namespace paimon { + +/// Merges the blob files of one data-evolution blob bunch that span multiple max sequence +/// number layers, resolving placeholder entries row by row. Aligned with Java's +/// BlobFallbackRecordReader / AllPlaceholdersRecordReader. +/// +/// A data-evolution partial update rewrites only the touched rows of a blob column; the new blob +/// file records every untouched row as a placeholder entry. Reading therefore needs, per row, the +/// value from the newest layer that holds a real (non-placeholder) entry: +/// +/// 1. The caller groups the blob files by max sequence number (one group per layer, newest +/// first) and, inside each group, orders them by first row id. Row id ranges the group's +/// files do not cover are represented by gap segments, which stand for all-placeholder rows. +/// 2. All groups span the same overall row id range, so with the same row-ranges selection +/// applied they yield the same number of rows and can be stepped in lockstep. +/// 3. Each output row takes the first group, in max-sequence order, whose row is not a +/// placeholder. Placeholder rows are identified by exact equality with the +/// BlobDefs::kPlaceholderSentinel bytes, emitted by the blob format reader when +/// BlobDefs::kEmitPlaceholderSentinelKey is set. +/// 4. A row that is a placeholder in every group degrades to a null blob: it keeps its +/// _ROW_ID, reports -1 as its _SEQUENCE_NUMBER, and returns null for every other field. +class BlobFallbackBatchReader : public BatchReader { + public: + /// One piece of a sequence group: either a reader over a single blob file (already wrapped + /// with the usual per-file mapping readers and row selection), or a virtual gap standing for + /// row ids the group's files do not cover. Segments must be ordered by ascending row id. + struct Segment { + /// File segment: emits the rows of one blob file. Null for a gap segment. + std::unique_ptr reader; + /// Gap segment only: the selected row ids the gap emits (all placeholders), as sorted + /// disjoint ranges. Must not be empty for a gap segment. + std::vector gap_selected_ranges; + }; + + /// `sequence_groups` must be ordered by descending max sequence number and contain at least + /// two groups (a single group needs no fallback). `read_schema` is the schema every file + /// reader emits; it must contain exactly one blob field and may additionally contain the + /// row-tracking fields _ROW_ID and _SEQUENCE_NUMBER (completed per file by + /// CompleteRowTrackingFieldsBatchReader), which stay correct for rows that are a + /// placeholder in every layer. + static Result> Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool); + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + void Close() override; + + std::shared_ptr GetReaderMetrics() const override; + + private: + /// A run of consecutive rows already fetched from one group: `array` rows + /// [offset, offset + length) for a file segment, or `length` placeholder rows for a gap + /// segment (array is null). + struct Chunk { + std::shared_ptr array; + int64_t offset = 0; + int64_t length = 0; + /// Gap chunk only: the row id of each of the `length` rows, filled when the read + /// schema contains _ROW_ID so all-placeholder rows can keep their row id. + std::vector gap_row_ids; + }; + + /// Read progress of one sequence group. Move-only, matching Segment's unique_ptr member. + struct GroupCursor { + GroupCursor() = default; + GroupCursor(const GroupCursor&) = delete; + GroupCursor& operator=(const GroupCursor&) = delete; + GroupCursor(GroupCursor&&) = default; + GroupCursor& operator=(GroupCursor&&) = default; + + std::vector segments; + size_t segment_idx = 0; + /// Position inside the current gap segment: index into gap_selected_ranges and the + /// number of rows already emitted from that range. + size_t gap_range_idx = 0; + int64_t gap_range_pos = 0; + /// Rows fetched from the current file segment but not yet consumed. + std::deque> pending; + int64_t pending_pos = 0; + }; + + BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool); + + /// Collects up to `want` rows from the group into chunks. Only the first group may come up + /// short (which defines the window size); any later group ending early is a misalignment. + Result FillWindow(size_t group_idx, int64_t want, std::vector* chunks); + + /// Flags each of the `row_count` window rows of the given chunks as placeholder or not. + Result> ComputePlaceholderFlags(const std::vector& chunks, + int64_t row_count) const; + + /// Assembles one output column by stitching, per run of rows choosing the same group, + /// slices of that group's chunks. For rows choosing no group (placeholder in every layer), + /// _ROW_ID is kept, _SEQUENCE_NUMBER becomes -1, and every other field becomes null. + Result> AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const; + + /// Assembles the _ROW_ID values of rows [run_start, run_end) from the given group's chunks; + /// gap chunks contribute their synthesized row ids. + Result> AssembleRowIdRun(const std::vector& chunks, + int64_t run_start, + int64_t run_end) const; + + std::vector groups_; + std::shared_ptr read_schema_; + const int32_t blob_field_idx_; + /// Index of _ROW_ID / _SEQUENCE_NUMBER in the read schema, -1 when not read. + const int32_t row_id_field_idx_; + const int32_t seq_num_field_idx_; + const int32_t read_batch_size_; + std::shared_ptr arrow_pool_; + bool closed_ = false; +}; + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp new file mode 100644 index 000000000..c1447aa92 --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp @@ -0,0 +1,358 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/range.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +/// "PH" stands for a placeholder row (the sentinel bytes emitted by the placeholder-aware blob +/// reader), std::nullopt for null. +using BlobRows = std::vector>; + +class BlobFallbackBatchReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); + read_schema_ = arrow::schema(struct_type_->fields()); + } + + static std::string Sentinel() { + return std::string(BlobDefs::PlaceholderSentinelView()); + } + + std::shared_ptr MakeBlobStruct(const BlobRows& rows) const { + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + } + + /// One segment of a group: File(rows) for a file segment, Gap(n) for a placeholder gap + /// of n selected rows (with synthetic row ids when the schema does not read them, or the + /// given ranges via GapRanges). + struct SegmentSpec { + std::vector gap_ranges; + std::optional file_rows; + static SegmentSpec Gap(int64_t rows) { + return SegmentSpec{{Range(0, rows - 1)}, std::nullopt}; + } + static SegmentSpec GapRanges(std::vector ranges) { + return SegmentSpec{std::move(ranges), std::nullopt}; + } + static SegmentSpec File(BlobRows rows) { + return SegmentSpec{{}, std::move(rows)}; + } + }; + + std::vector MakeGroup(const std::vector& specs, + int32_t file_batch_size) const { + std::vector segments; + for (const auto& spec : specs) { + if (spec.file_rows) { + auto reader = std::make_unique(MakeBlobStruct(*spec.file_rows), + struct_type_, file_batch_size); + segments.push_back(BlobFallbackBatchReader::Segment{std::move(reader), {}}); + } else { + segments.push_back(BlobFallbackBatchReader::Segment{nullptr, spec.gap_ranges}); + } + } + return segments; + } + + /// Runs the fallback over the groups with several batch sizes and compares to expected rows. + void CheckFallback(const std::vector>& group_specs, + const BlobRows& expected_rows) const { + auto expected_array = MakeBlobStruct(expected_rows); + for (auto batch_size : arrow::internal::Iota(1, 8)) { + for (auto file_batch_size : {1, 3, 1024}) { + std::vector> groups; + groups.reserve(group_specs.size()); + for (const auto& specs : group_specs) { + groups.push_back(MakeGroup(specs, file_batch_size)); + } + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + auto expected_chunk_array = std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + + protected: + std::shared_ptr pool_; + std::shared_ptr struct_type_; + std::shared_ptr read_schema_; +}; + +TEST_F(BlobFallbackBatchReaderTest, TestBasicFallback) { + // newer layer updates row 1 only; rows 0 and 2 fall back to the older layer + CheckFallback( + {{SegmentSpec::File({"PH", "u1", "PH"})}, {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "u1", "b2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestGapPadding) { + // the newer layer only covers rows 2-3; the gaps stand for placeholders + CheckFallback({{SegmentSpec::Gap(2), SegmentSpec::File({"u2", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "b1", "u2", "b3"}); + // trailing gap + CheckFallback({{SegmentSpec::File({"PH", "u1"}), SegmentSpec::Gap(2)}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "u1", "b2", "b3"}); + // middle gap between two files of one layer + CheckFallback({{SegmentSpec::File({"u0"}), SegmentSpec::Gap(2), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"u0", "b1", "b2", "u3"}); + // a gap segment covering multiple disjoint selected ranges + CheckFallback({{SegmentSpec::GapRanges({Range(0, 0), Range(2, 2)}), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b2", "b3"})}}, + {"b0", "b2", "u3"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestAllPlaceholdersBecomesNull) { + // a row that is a placeholder in every layer degrades to null + CheckFallback({{SegmentSpec::File({"PH", "PH"})}, {SegmentSpec::File({"b0", "PH"})}}, + {"b0", std::nullopt}); + CheckFallback({{SegmentSpec::Gap(2)}, {SegmentSpec::File({"PH", "PH"})}}, + {std::nullopt, std::nullopt}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestNullIsNotPlaceholder) { + // a real null in a newer layer wins: null means "updated to null", not "not updated" + CheckFallback({{SegmentSpec::File({std::nullopt, "u1"})}, {SegmentSpec::File({"b0", "b1"})}}, + {std::nullopt, "u1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestSentinelPrefixedValueIsNotPlaceholder) { + // placeholders are identified by exact equality with the sentinel bytes only: a real value + // that merely starts with them passes through unchanged, whether it falls back or wins as + // the newest layer + std::string prefixed = Sentinel() + "suffix"; + CheckFallback({{SegmentSpec::File({"PH", "u1"})}, {SegmentSpec::File({prefixed, "b1"})}}, + {prefixed, "u1"}); + CheckFallback({{SegmentSpec::File({prefixed, "PH"})}, {SegmentSpec::File({"b0", "b1"})}}, + {prefixed, "b1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestThreeLayers) { + CheckFallback({{SegmentSpec::File({"PH", "PH", "u2"})}, + {SegmentSpec::File({"PH", "m1", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "m1", "u2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestLayeredFilesAndGaps) { + // mirrors the compacted-sequence-groups shape: layers partially cover [0, 9] + CheckFallback( + {{SegmentSpec::Gap(6), SegmentSpec::File({"u66", "PH"}), SegmentSpec::Gap(1), + SegmentSpec::File({"u69"})}, + {SegmentSpec::File({"u40", "PH", "PH", "PH"}), SegmentSpec::Gap(4), + SegmentSpec::File({"u48", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9"})}}, + {"u40", "b1", "b2", "b3", "b4", "b5", "u66", "b7", "u48", "u69"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestRowTrackingFieldsPreserved) { + // Row-tracking projections: resolved rows keep their layer's row id and sequence number; + // an all-placeholder row keeps its row id (here provided by the newest group's gap + // segment), reports -1 as its sequence number, and degrades the blob to null. Covers the + // schema variants {blob, _ROW_ID, _SEQUENCE_NUMBER}, {blob, _ROW_ID} and + // {blob, _SEQUENCE_NUMBER}. + struct RowSpec { + std::optional blob; + int64_t row_id; + int64_t seq_num; + }; + for (bool with_row_id : {true, false}) { + for (bool with_seq_num : {true, false}) { + if (!with_row_id && !with_seq_num) { + continue; + } + arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", true)}; + if (with_row_id) { + fields.push_back(SpecialFields::RowId().field_); + } + if (with_seq_num) { + fields.push_back(SpecialFields::SequenceNumber().field_); + } + auto struct_type = arrow::struct_(fields); + auto schema = arrow::schema(fields); + + auto make_rows = [&](const std::vector& rows) { + std::vector> field_builders = { + std::make_shared()}; + for (size_t i = 1; i < fields.size(); i++) { + field_builders.push_back(std::make_shared()); + } + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row.blob) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row.blob == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row.blob->data(), row.blob->size()).ok()); + } + int32_t next_field = 1; + if (with_row_id) { + auto builder = static_cast( + struct_builder.field_builder(next_field++)); + EXPECT_TRUE(builder->Append(row.row_id).ok()); + } + if (with_seq_num) { + auto builder = static_cast( + struct_builder.field_builder(next_field)); + EXPECT_TRUE(builder->Append(row.seq_num).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + }; + + for (auto batch_size : arrow::internal::Iota(1, 5)) { + for (auto file_batch_size : {1, 1024}) { + // newest layer (seq 20) covers only row 2; rows 0-1 are a gap + std::vector newest; + newest.push_back(BlobFallbackBatchReader::Segment{nullptr, {Range(0, 1)}}); + newest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique(make_rows({{"u2", 2, 20}}), + struct_type, file_batch_size), + {}}); + // oldest layer (seq 10) covers rows 0-2, row 1 is a placeholder there too + std::vector oldest; + oldest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique( + make_rows({{"b0", 0, 10}, {"PH", 1, 10}, {"PH", 2, 10}}), struct_type, + file_batch_size), + {}}); + std::vector> groups; + groups.push_back(std::move(newest)); + groups.push_back(std::move(oldest)); + + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFallbackBatchReader::Create(std::move(groups), schema, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + + // row 0 falls back to seq 10, row 1 is all-placeholder (null blob, row id + // kept, seq -1), row 2 takes seq 20 + auto expected_array = + make_rows({{"b0", 0, 10}, {std::nullopt, 1, -1}, {"u2", 2, 20}}); + auto expected_chunk_array = + std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "with_row_id=" << with_row_id << " with_seq_num=" << with_seq_num + << " batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + } +} + +TEST_F(BlobFallbackBatchReaderTest, TestMisalignedGroupsFail) { + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"PH", "u1", "PH"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b0", "b1"})}, 1024)); + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, 1024, pool_)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "same number of rows"); +} + +TEST_F(BlobFallbackBatchReaderTest, TestCreateValidation) { + // a single group needs no fallback + std::vector> single_group; + single_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(single_group), read_schema_, 1024, pool_), + "at least two sequence groups"); + + // the read schema must contain a blob field + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b1"})}, 1024)); + auto plain_schema = + arrow::schema({arrow::field("not_blob", arrow::large_binary(), /*nullable=*/true)}); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(groups), plain_schema, 1024, pool_), + "should contain a blob field"); + + // groups must not be empty + std::vector> with_empty_group; + with_empty_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_group.emplace_back(); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_group), read_schema_, 1024, pool_), + "should not be empty"); + + // a gap segment must cover at least one selected row id + std::vector> with_empty_gap; + with_empty_gap.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_gap.push_back(MakeGroup({SegmentSpec::GapRanges({})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_gap), read_schema_, 1024, pool_), + "at least one selected row id"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index a4e49ecbb..0c3a45578 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -41,8 +41,13 @@ Result> DataEvolutionFileReader::Create return Status::Invalid( "read schema, row offsets and field offsets must have the same size"); } - if (readers.size() <= 1) { - return Status::Invalid("readers size is supposed to be more than 1"); + if (readers.empty()) { + return Status::Invalid("readers must not be empty"); + } + for (int32_t reader_offset : reader_offsets) { + if (reader_offset >= static_cast(readers.size())) { + return Status::Invalid("reader offset is out of range of readers"); + } } return std::unique_ptr( new DataEvolutionFileReader(std::move(readers), read_schema, read_batch_size, diff --git a/src/paimon/common/reader/data_evolution_file_reader.h b/src/paimon/common/reader/data_evolution_file_reader.h index 2ba5853ef..36bb2641f 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.h +++ b/src/paimon/common/reader/data_evolution_file_reader.h @@ -26,10 +26,12 @@ #include "paimon/result.h" namespace paimon { -/// This is a union reader which contains multiple inner readers. +/// This is a union reader which contains one or more inner readers. /// /// This reader, assembling multiple reader into one big and great reader. The row it produces -/// also come from the readers it contains. +/// also come from the readers it contains. A single inner reader is a valid degenerate case +/// (e.g. a blob-only write): the union still maps its fields into the read schema order and +/// null fills unmatched read fields. /// /// For example, the expected schema for this reader is : int, int, string, int, string, int.(Total /// 6 fields) It contains three inner readers, we call them reader0, reader1 and reader2. diff --git a/src/paimon/common/reader/data_evolution_file_reader_test.cpp b/src/paimon/common/reader/data_evolution_file_reader_test.cpp index 4eccca7ab..3f18448c9 100644 --- a/src/paimon/common/reader/data_evolution_file_reader_test.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader_test.cpp @@ -163,6 +163,20 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) { reader_offsets, field_offsets, pool_), "read schema, row offsets and field offsets must have the same size"); } + { + arrow::FieldVector read_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::int32()), + }; + auto read_schema = arrow::schema(read_fields); + std::vector reader_offsets = {0, 0, 1, 1}; + std::vector field_offsets = {0, 1, 1, 0}; + ASSERT_NOK_WITH_MSG(DataEvolutionFileReader::Create({}, read_schema, /*read_batch_size=*/10, + reader_offsets, field_offsets, pool_), + "readers must not be empty"); + } { std::vector> readers; readers.push_back(nullptr); @@ -179,7 +193,7 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) { ASSERT_NOK_WITH_MSG( DataEvolutionFileReader::Create(std::move(readers), read_schema, /*read_batch_size=*/10, reader_offsets, field_offsets, pool_), - "readers size is supposed to be more than 1"); + "reader offset is out of range of readers"); } } diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 4b9c68e3a..3886dba9f 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -211,11 +211,11 @@ Result AppendOnlyWriter::GetDataFileWriterFacto AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const { + const std::optional>& write_cols, bool write_placeholder) const { std::shared_ptr path_factory = path_factory_; return std::make_shared(options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, - memory_pool_); + write_placeholder, memory_pool_); } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( @@ -223,8 +223,16 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri // Multiple blob fields are supported. Each blob field gets its own rolling file writer // via MultipleBlobFileWriter. auto blob_schema = schemas.blob_schema; + // A data-evolution write touching only blob columns is a partial update: its rows may mark + // untouched blobs with the placeholder sentinel (see BlobDefs::kWritePlaceholderKey). Any + // write that also carries non-blob columns stores blob bytes verbatim. The gate cannot tell + // a first write from an update, so a blob-only first write also runs under the sentinel + // channel: a user value equal to the sentinel is then persisted as a placeholder entry with + // no older layer to resolve it, and reading fails loudly rather than returning wrong bytes. + bool write_placeholder = + options_.DataEvolutionEnabled() && schemas.main_schema->num_fields() == 0; MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator = - [this, blob_schema](const std::string& blob_field_name) + [this, blob_schema, write_placeholder](const std::string& blob_field_name) -> Result< std::unique_ptr>>> { // Create a single-field schema for this blob field @@ -236,7 +244,7 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri auto single_field_schema = arrow::schema({field}); std::vector write_cols = {blob_field_name}; auto single_blob_file_writer_factory = - GetBlobFileWriterFactory(single_field_schema, write_cols); + GetBlobFileWriterFactory(single_field_schema, write_cols, write_placeholder); return std::make_unique>>( options_.GetBlobTargetFileSize(), /*target_file_row_num=*/std::numeric_limits::max(), diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 80bc18b88..a00f1e443 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -108,7 +108,7 @@ class AppendOnlyWriter : public BatchWriter { WriterFactory GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const; + const std::optional>& write_cols, bool write_placeholder) const; Status TrySyncLatestCompaction(bool blocking); Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp index aca03e7ab..c7a748270 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.cpp +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -17,8 +17,11 @@ #include "paimon/core/io/blob_data_file_writer_factory.h" #include +#include +#include #include +#include "paimon/common/data/blob_defs.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/manifest/file_source.h" @@ -33,20 +36,28 @@ BlobDataFileWriterFactory::BlobDataFileWriterFactory( const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, - const std::shared_ptr& path_factory, + const std::shared_ptr& path_factory, bool write_placeholder, const std::shared_ptr& pool) : DataFileWriterFactory(options, schema_id, pool), file_schema_(file_schema), write_cols_(write_cols), seq_num_counter_(seq_num_counter), - path_factory_(path_factory) {} + path_factory_(path_factory), + write_placeholder_(write_placeholder) {} Result>>> BlobDataFileWriterFactory::CreateWriter() const { std::shared_ptr seq_num_counter = options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; + std::map format_options = options_.ToMap(); + // The placeholder channel is internal: strip user-supplied blob.internal.* table options so + // only the writer's own decision below can enable it. + BlobDefs::EraseInternalPlaceholderOptions(&format_options); + if (write_placeholder_) { + format_options[BlobDefs::kWritePlaceholderKey] = "true"; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, - FileFormatFactory::Get("blob", options_.ToMap())); + FileFormatFactory::Get("blob", format_options)); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema_, /*create_stats_extractor=*/true)); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h index e79d42e13..3ff813f97 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.h +++ b/src/paimon/core/io/blob_data_file_writer_factory.h @@ -43,12 +43,15 @@ class BlobDataFileWriterFactory : public DataFileWriterFactory, public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { public: + /// `write_placeholder` marks a data-evolution partial-update write: the blob format + /// writer is created with BlobDefs::kWritePlaceholderKey and persists placeholder + /// sentinel rows as placeholder entries. BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + bool write_placeholder, const std::shared_ptr& pool); Result>>> CreateWriter() const override; @@ -58,6 +61,7 @@ class BlobDataFileWriterFactory std::optional> write_cols_; std::shared_ptr seq_num_counter_; std::shared_ptr path_factory_; + bool write_placeholder_ = false; }; } // namespace paimon diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp index 29b610b16..896efcf82 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp @@ -26,34 +26,46 @@ #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/object_utils.h" namespace paimon { CompleteRowTrackingFieldsBatchReader::CompleteRowTrackingFieldsBatchReader( std::unique_ptr&& reader, const std::optional& first_row_id, - int64_t snapshot_id, const std::shared_ptr& pool) + int64_t snapshot_id, const std::optional>& file_field_names, + const std::shared_ptr& pool) : first_row_id_(first_row_id), snapshot_id_(snapshot_id), + file_field_names_(file_field_names), arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)) {} Status CompleteRowTrackingFieldsBatchReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, - arrow::ImportSchema(c_file_schema.get())); + // The physical fields of the file decide which special fields must be stripped from the + // format reader's schema: a format without a self-describing file schema (e.g. blob) + // declares them via file_field_names_, self-describing formats are queried directly. + std::vector file_field_names; + if (file_field_names_) { + file_field_names = file_field_names_.value(); + } else { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, + reader_->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, + arrow::ImportSchema(c_file_schema.get())); + file_field_names = file_schema->field_names(); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(read_schema)); read_schema_ = arrow_schema; int32_t row_id_idx = arrow_schema->GetFieldIndex(SpecialFields::RowId().Name()); - if (row_id_idx != -1 && file_schema->GetFieldIndex(SpecialFields::RowId().Name()) == -1) { - // read special fields but file not exist, remove special fields to format reader + if (row_id_idx != -1 && + !ObjectUtils::Contains(file_field_names, SpecialFields::RowId().Name())) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(row_id_idx)); } int32_t sequence_id_idx = arrow_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); if (sequence_id_idx != -1 && - file_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()) == -1) { - // read special fields but file not exist, remove special fields to format reader + !ObjectUtils::Contains(file_field_names, SpecialFields::SequenceNumber().Name())) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(sequence_id_idx)); } ArrowSchema c_schema; diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.h b/src/paimon/core/io/complete_row_tracking_fields_reader.h index 2aa535d79..607e3ca5c 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.h +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.h @@ -31,10 +31,13 @@ namespace paimon { // Precondition: read_schema has special fields class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { public: - CompleteRowTrackingFieldsBatchReader(std::unique_ptr&& reader, - const std::optional& first_row_id, - int64_t snapshot_id, - const std::shared_ptr& pool); + /// `file_field_names` declares the physical fields of a file whose format has no + /// self-describing schema (e.g. blob); when nullopt the file schema is queried from the + /// inner reader via GetFileSchema(). + CompleteRowTrackingFieldsBatchReader( + std::unique_ptr&& reader, const std::optional& first_row_id, + int64_t snapshot_id, const std::optional>& file_field_names, + const std::shared_ptr& pool); Result> GetFileSchema() const override { return Status::Invalid( @@ -80,6 +83,7 @@ class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { private: std::optional first_row_id_; int64_t snapshot_id_ = -1; + std::optional> file_field_names_; std::shared_ptr arrow_pool_; std::shared_ptr read_schema_; std::unique_ptr reader_; diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp index 2ca1be8a6..1a80b08ec 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp @@ -17,7 +17,10 @@ #include "paimon/core/io/complete_row_tracking_fields_reader.h" #include +#include +#include #include +#include #include "arrow/api.h" #include "arrow/array/array_base.h" @@ -44,7 +47,8 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test { std::make_unique(src_array, src_array->type(), batch_size); auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), first_row_id, snapshot_id, pool_); + std::move(file_batch_reader), first_row_id, snapshot_id, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK(complete_row_tracking_fields_reader->SetReadSchema( @@ -63,12 +67,14 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test { void CheckSetReadSchema( const std::shared_ptr& file_schema, const std::shared_ptr& read_schema, - const std::shared_ptr& expected_schema_for_inner_reader) const { + const std::shared_ptr& expected_schema_for_inner_reader, + const std::optional>& file_field_names = std::nullopt) const { auto file_batch_reader = std::make_unique( /*data=*/nullptr, arrow::struct_(file_schema->fields()), /*read_batch_size=*/1); auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1, pool_); + std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1, + file_field_names, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( @@ -130,6 +136,33 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchema) { } } +TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchemaWithDeclaredFileFields) { + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + arrow::field("_ROW_ID", arrow::int64()), + arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + }; + { + // declared physical fields take precedence over the inner reader's self-described + // schema: the special fields are stripped even though the mock's file schema + // contains them + auto file_schema = arrow::schema(fields); + auto read_schema = arrow::schema(fields); + auto expected_schema_for_inner_reader = arrow::schema({fields[0], fields[1]}); + CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader, + std::vector{"f0", "f1"}); + } + { + // declared physical fields containing the special fields keep them for the inner reader + auto file_schema = arrow::schema({fields[0], fields[1]}); + auto read_schema = arrow::schema(fields); + auto expected_schema_for_inner_reader = read_schema; + CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader, + std::vector{"f0", "f1", "_ROW_ID", "_SEQUENCE_NUMBER"}); + } +} + TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestWithNoRowTrackingFields) { arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), @@ -342,7 +375,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidWithReadNonExistFiel auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( @@ -378,7 +412,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNextBatchBeforeSetRe auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ASSERT_NOK_WITH_MSG(complete_row_tracking_fields_reader->NextBatchWithBitmap(), "in CompleteRowTrackingFieldsBatchReader SetReadSchema is supposed to be " "called before NextBatch"); @@ -408,7 +443,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNullFirstRowId) { auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index ab323a235..4cea81edf 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -345,7 +345,8 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptr> raw_readers, split_read_->CreateRawFileReaders(partition_, {file}, read_schema_, /*predicate=*/nullptr, dv_factory_, - /*row_ranges=*/std::nullopt, data_file_path_factory_)); + /*row_ranges=*/std::nullopt, data_file_path_factory_, + /*extra_format_options=*/{})); if (raw_readers.size() != 1) { return Status::Invalid("Unexpected, CreateSstFileFromDataFile only create single reader"); } diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index cee6fd0b3..437f062f1 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -24,6 +24,7 @@ #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" @@ -75,7 +76,8 @@ Result>> AbstractSplitRead::CreateR const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const { + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const { if (data_files.empty()) { return std::vector>(); } @@ -89,7 +91,7 @@ Result>> AbstractSplitRead::CreateR auto data_file_path = data_file_path_factory->ToPath(file); PAIMON_ASSIGN_OR_RAISE(std::string data_file_identifier, file->FileFormat()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, - PrepareReaderBuilder(data_file_identifier)); + PrepareReaderBuilder(data_file_identifier, extra_format_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_reader, CreateFieldMappingReader(data_file_path, file, partition, reader_builder.get(), @@ -120,9 +122,17 @@ Result> AbstractSplitRead::ApplyPredicateFilterIfNe } Result> AbstractSplitRead::PrepareReaderBuilder( - const std::string& format_identifier) const { + const std::string& format_identifier, + const std::map& extra_format_options) const { + std::map format_options = options_.ToMap(); + // The blob placeholder channels are internal: strip user-supplied blob.internal.* table + // options so only the internal read path can enable them through extra_format_options. + BlobDefs::EraseInternalPlaceholderOptions(&format_options); + for (const auto& [key, value] : extra_format_options) { + format_options[key] = value; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format, - FileFormatFactory::Get(format_identifier, options_.ToMap())); + FileFormatFactory::Get(format_identifier, format_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, file_format->CreateReaderBuilder(options_.GetReadBatchSize())); reader_builder->WithMemoryPool(pool_); @@ -206,8 +216,15 @@ Result> AbstractSplitRead::CreateFieldMappingRe file_reader, ApplyVariantShreddingReaderIfNeeded(std::move(file_reader), read_schema)); } if (NeedCompleteRowTrackingFields(options_.RowTrackingEnabled(), read_schema)) { + // A blob file has no self-describing schema: its physical fields are declared by the + // file meta's write cols instead of queried from the format reader. + std::optional> file_field_names; + if (file_format_identifier == "blob") { + file_field_names = file_meta->write_cols; + } file_reader = std::make_unique( - std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, pool_); + std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, + file_field_names, pool_); } const auto& predicate = field_mapping->non_partition_info.non_partition_filter; auto all_data_schema = DataField::ConvertDataFieldsToArrowSchema(data_schema->Fields()); diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index 89c94fbf4..933c284d5 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -60,12 +61,16 @@ class AbstractSplitRead : public SplitRead { public: ~AbstractSplitRead() override = default; + /// `extra_format_options` are merged over the table options when building the format + /// reader, e.g. to switch the blob format reader into placeholder-aware mode for the + /// data-evolution blob fallback read path. Result>> CreateRawFileReaders( const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const; + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const; protected: AbstractSplitRead(const std::shared_ptr& path_factory, @@ -95,7 +100,8 @@ class AbstractSplitRead : public SplitRead { private: Result> PrepareReaderBuilder( - const std::string& format_identifier) const; + const std::string& format_identifier, + const std::map& extra_format_options) const; Result> CreateFileBatchReader( const std::string& file_format_identifier, const std::string& data_file_path, diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 01ed2f31f..d0c5640e8 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -16,6 +16,8 @@ #include "paimon/core/operation/data_evolution_split_read.h" +#include +#include #include #include #include @@ -28,10 +30,12 @@ #include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" #include "paimon/common/catalog/catalog_context.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h" #include "paimon/common/global_index/complete_index_score_batch_reader.h" +#include "paimon/common/reader/blob_fallback_batch_reader.h" #include "paimon/common/reader/blob_view_resolving_batch_reader.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -45,71 +49,61 @@ #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/utils/blob_view_lookup.h" namespace paimon { +int64_t DataEvolutionSplitRead::BlobBunch::RowCount() const { + if (files_.empty()) { + return 0; + } + if (!has_row_ids_selection_) { + // Add enforces the union range to be contiguous + return union_end_row_id_ - union_first_row_id_; + } + // with a row-ids selection the scan may have pruned files, leaving holes in the union + int64_t row_count = 0; + for (const auto& range : Range::SortAndMergeOverlap(ranges_, /*adjacent=*/true)) { + row_count += range.Count(); + } + return row_count; +} + Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr& file) { if (!BlobUtils::IsBlobFile(file->file_name)) { return Status::Invalid("Only blob file can be added to a blob bunch."); } PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); - if (first_row_id == latest_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with same first row id should have decreasing sequence number."); - } - // for files with the same first row id, file with larger sequence_number will be chosen, - // other files will be skipped - return Status::OK(); - } if (!files_.empty()) { - if (has_row_ids_selection_) { - // for the case: - // snapshot 1: blob0 [0, 9] - // snapshot 2: blob1 [0, 4] + blob2 [5, 9] - // when selected row id is {5}, only blob0 and blob2 is reserved in scan process, as - // blob1 has no intersect with {5} - // BlobBunch will first add blob0 [0, 9] - // then when it comes to blob2 [5, 9], blob0 will be removed as it has smaller sequence - // number - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number > latest_max_sequence_number_) { - row_count_ -= files_.back()->row_count; - files_.pop_back(); - } else { - return Status::OK(); - } - } - } else { - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with overlapping row id should have decreasing sequence " - "number."); - } - // for files with overlapping, if the file with smaller sequence_number is chosen, - // there will not exist file with larger sequence_number - return Status::OK(); - } else if (first_row_id > expected_next_first_row_id_) { - return Status::Invalid( - fmt::format("Blob file first row id should be continuous, expect {} but got {}", - expected_next_first_row_id_, first_row_id)); - } - } - if (!files_.empty()) { - // Blob files for the same field may span schema ids. - if (file->write_cols != files_[0]->write_cols) { - return Status::Invalid( - "All files in a blob bunch should have the same write columns."); - } + // Blob files for the same field may span schema ids. + if (file->write_cols != files_[0]->write_cols) { + return Status::Invalid("All files in a blob bunch should have the same write columns."); } } - row_count_ += file->row_count; - if (row_count_ > expected_row_count_) { + // files sharing a max sequence number form one layer, whose row id ranges must be disjoint; + // overlaps across layers are the expected shape of partial updates and are kept for the + // row-level placeholder fallback + auto [layer_iter, layer_inserted] = + sequence_group_end_.try_emplace(file->max_sequence_number, 0); + if (!layer_inserted && first_row_id < layer_iter->second) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping row id " + "ranges: file {} (max sequence number {}) starts at row id {} before the previous " + "file's end {}", + file->file_name, file->max_sequence_number, first_row_id, layer_iter->second)); + } + if (!has_row_ids_selection_ && !files_.empty() && first_row_id > union_end_row_id_) { + // a hole no layer covers cannot be aligned with the data files return Status::Invalid( - fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + fmt::format("Blob file first row id should be continuous, expect {} but got {}", + union_end_row_id_, first_row_id)); } + int64_t end_row_id = first_row_id + file->row_count; + layer_iter->second = end_row_id; + union_first_row_id_ = std::min(union_first_row_id_, first_row_id); + union_end_row_id_ = std::max(union_end_row_id_, end_row_id); + ranges_.emplace_back(first_row_id, end_row_id - 1); files_.push_back(file); - latest_max_sequence_number_ = file->max_sequence_number; - latest_first_row_id_ = first_row_id; - expected_next_first_row_id_ = latest_first_row_id_ + file->row_count; + if (!has_row_ids_selection_ && expected_row_count_ >= 0 && RowCount() > expected_row_count_) { + return Status::Invalid( + fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + } return Status::OK(); } DataEvolutionSplitRead::DataEvolutionSplitRead( @@ -233,7 +227,8 @@ Result> DataEvolutionSplitRead::CreateBlobViewReade std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema, /*predicate=*/nullptr, /*dv_factory=*/nullptr, - /*row_ranges=*/std::nullopt, data_file_path_factory)); + /*row_ranges=*/std::nullopt, data_file_path_factory, + /*extra_format_options=*/{})); auto batch_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -312,7 +307,8 @@ Result> DataEvolutionSplitRead::InnerCreateReader( std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_, /*predicate=*/nullptr, - /*dv_factory=*/nullptr, row_ranges, data_file_path_factory)); + /*dv_factory=*/nullptr, row_ranges, data_file_path_factory, + /*extra_format_options=*/{})); assert(raw_file_readers.size() == 1); sub_readers.push_back(std::move(raw_file_readers[0])); } else { @@ -488,18 +484,30 @@ Result> DataEvolutionSplitRead::CreateU if (!read_fields_in_file.empty()) { // create new FieldMappingReader for read partial fields auto file_read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields_in_file); - PAIMON_ASSIGN_OR_RAISE(std::vector> file_readers, - CreateRawFileReaders(partition, bunch->Files(), file_read_schema, - /*predicate=*/nullptr, /*dv_factory=*/{}, - row_ranges, data_file_path_factory)); - if (file_readers.size() == 1) { - file_batch_readers[file_idx] = std::move(file_readers[0]); + auto blob_bunch = std::dynamic_pointer_cast(bunch); + if (blob_bunch && !blob_bunch->SequentialReadOptimize()) { + // blob files span multiple max sequence number layers: placeholder entries of + // newer layers must fall back row by row to older layers + PAIMON_ASSIGN_OR_RAISE( + file_batch_readers[file_idx], + CreateBlobFallbackReader(partition, blob_bunch->Files(), file_read_schema, + row_ranges, data_file_path_factory)); } else { - auto raw_readers = - ObjectUtils::MoveVector>(std::move(file_readers)); - // Concat multiple blob files that map to the same data file. - file_batch_readers[file_idx] = - std::make_unique(std::move(raw_readers), pool_); + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, bunch->Files(), file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, + /*extra_format_options=*/{})); + if (file_readers.size() == 1) { + file_batch_readers[file_idx] = std::move(file_readers[0]); + } else { + auto raw_readers = ObjectUtils::MoveVector>( + std::move(file_readers)); + // Concat multiple blob files that map to the same data file. + file_batch_readers[file_idx] = + std::make_unique(std::move(raw_readers), pool_); + } } } } @@ -509,6 +517,98 @@ Result> DataEvolutionSplitRead::CreateU field_offsets, pool_); } +namespace { +/// Selected row ids in [from, to] as sorted disjoint ranges: the whole range without a +/// selection, otherwise its intersection with the (possibly overlapping) selected ranges. +std::vector SelectedRangesInRange(int64_t from, int64_t to, + const std::optional>& row_ranges) { + if (!row_ranges) { + return {Range(from, to)}; + } + std::vector selected; + Range gap_range(from, to); + for (const auto& range : Range::SortAndMergeOverlap(row_ranges.value(), /*adjacent=*/true)) { + std::optional intersection = Range::Intersection(gap_range, range); + if (intersection) { + selected.push_back(*intersection); + } + } + return selected; +} +} // namespace + +Result> DataEvolutionSplitRead::CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const { + int64_t union_first_row_id = std::numeric_limits::max(); + int64_t union_last_row_id = std::numeric_limits::min(); + using FileWithFirstRowId = std::pair>; + std::map, std::greater<>> sequence_groups; + for (const auto& file : files) { + PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); + union_first_row_id = std::min(union_first_row_id, first_row_id); + union_last_row_id = std::max(union_last_row_id, first_row_id + file->row_count - 1); + sequence_groups[file->max_sequence_number].emplace_back(first_row_id, file); + } + // the blob format reader must emit placeholder sentinels instead of failing on them + const std::map blob_format_options = { + {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + std::vector> groups; + groups.reserve(sequence_groups.size()); + for (auto& [max_sequence_number, group_files] : sequence_groups) { + std::stable_sort(group_files.begin(), group_files.end(), + [](const FileWithFirstRowId& f1, const FileWithFirstRowId& f2) { + return f1.first < f2.first; + }); + std::vector segments; + // pad row ids this layer does not cover with placeholder gaps, so that every layer spans + // the same union range and the groups can be stepped in lockstep + int64_t next_row_id = union_first_row_id; + for (const auto& [first_row_id, file] : group_files) { + if (first_row_id < next_row_id) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping " + "row id ranges: file {} (max sequence number {}) starts at row id {} before " + "the previous file's end {}", + file->file_name, max_sequence_number, first_row_id, next_row_id)); + } + if (first_row_id > next_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, first_row_id - 1, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, {file}, file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, blob_format_options)); + if (file_readers.size() != 1) { + return Status::Invalid("Unexpected: blob fallback file reader was skipped."); + } + segments.push_back(BlobFallbackBatchReader::Segment{std::move(file_readers[0]), {}}); + next_row_id = first_row_id + file->row_count; + } + if (next_row_id <= union_last_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, union_last_row_id, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + groups.push_back(std::move(segments)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_reader, + BlobFallbackBatchReader::Create(std::move(groups), file_read_schema, + options_.GetReadBatchSize(), pool_)); + return std::move(fallback_reader); +} + Result DataEvolutionSplitRead::Match(const std::shared_ptr& split, bool force_keep_delete) const { return true; diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index c3ed5033f..87b602602 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -55,7 +57,8 @@ struct DeletionFile; /// Readers Overview: (ConcatBatchReader across /// splits)->(BlobViewResolvingBatchReader)->(CompleteIndexScoreBatchReader)-> /// CompleteRowKindBatchReader->(PredicateBatchReader) -/// ->ConcatBatchReader across files->DataEvolutionFileReader->(ConcatBatchReader across blob files) +/// ->ConcatBatchReader across files->DataEvolutionFileReader +/// ->(ConcatBatchReader across blob files | BlobFallbackBatchReader across blob sequence layers) /// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader) /// ->(MapSharedShreddingFileReader) /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader @@ -107,26 +110,43 @@ class DataEvolutionSplitRead : public AbstractSplitRead { std::vector> data_files_; }; + /// All blob files of one blob field in a merge group. Unlike data files, every file is kept + /// (aligned with Java's BlobFileBunch): files whose max sequence numbers differ form layers + /// of a data-evolution partial update, where newer layers record untouched rows as + /// placeholder entries and reading falls back row by row to older layers. + /// Files must be added ordered by first row id ascending (then max sequence number + /// descending), as produced by MergeRangesAndSort. class BlobBunch : public FieldBunch { public: explicit BlobBunch(int64_t expected_row_count, bool has_row_ids_selection) : expected_row_count_(expected_row_count), has_row_ids_selection_(has_row_ids_selection) {} - int64_t RowCount() const override { - return row_count_; - } + /// Number of distinct row ids covered by the added files. Without a row-ids selection + /// the covered range is contiguous (enforced by Add) and matches the data files. + int64_t RowCount() const override; const std::vector>& Files() const override { return files_; } Status Add(const std::shared_ptr& file); + /// True when every file shares one max sequence number, so the files are read + /// sequentially without the fallback merge. A lone layer is expected to hold no + /// placeholder entries; if one does (a user value equal to the placeholder sentinel + /// written by a blob-only first write), the strict blob reader rejects the read, since + /// no older layer exists to resolve it. + bool SequentialReadOptimize() const { + return sequence_group_end_.size() <= 1; + } private: int64_t expected_row_count_ = -1; - int64_t latest_first_row_id_ = -1; - int64_t expected_next_first_row_id_ = -1; - int64_t latest_max_sequence_number_ = -1; - int64_t row_count_ = 0; bool has_row_ids_selection_ = false; + int64_t union_first_row_id_ = std::numeric_limits::max(); + /// Exclusive end of the union row id range covered so far. + int64_t union_end_row_id_ = std::numeric_limits::min(); + /// Per max sequence number: exclusive end of the last added range, to reject + /// overlapping files within one layer. + std::map sequence_group_end_; + std::vector ranges_; std::vector> files_; }; @@ -164,6 +184,16 @@ class DataEvolutionSplitRead : public AbstractSplitRead { const std::vector>& need_merge_files, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; + + /// Builds the row-level fallback reader for a blob bunch spanning multiple max sequence + /// number layers: groups the files by max sequence number, pads uncovered row id ranges of + /// each layer with placeholder gap segments, and resolves each row to the newest + /// non-placeholder layer. See BlobFallbackBatchReader. + Result> CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const; }; } // namespace paimon diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index aa6f9c9e1..4cf3a664b 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -145,72 +145,88 @@ TEST_F(DataEvolutionSplitReadTest, TestAddNonBlobFileInvalid) { TEST_F(DataEvolutionSplitReadTest, TestAddBlobWithSameFirstRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/1, + /*max_sequence_number=*/3, /*write_cols=*/std::optional>({"blob_col"})); - auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + auto blob_full_tail = + CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); + auto blob_short_tail = + CreateBlobFile("blob3", /*first_row_id=*/0, /*row_count=*/50, + /*max_sequence_number=*/1, + /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), - "Blob file with same first row id should have decreasing sequence number."); + // Files with the same first row id and lower sequence numbers are older layers of a + // partial update; they are kept for the row-level placeholder fallback, whether they + // cover the same range or only a shorter prefix of it. + ASSERT_OK(blob_bunch->Add(blob_full_tail)); + ASSERT_OK(blob_bunch->Add(blob_short_tail)); + + ASSERT_EQ(blob_bunch->Files().size(), 3); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_full_tail); + ASSERT_EQ(blob_bunch->Files()[2], blob_short_tail); + ASSERT_EQ(blob_bunch->RowCount(), 100); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithSameFirstRowIdAndLowerSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with same firstRowId and lower sequence number should be ignored + // Overlapping layers with different sequence numbers are kept for the fallback. ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/1, + /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with overlapping row id and lower sequence number should be ignored ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdInSameLayer) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG( - blob_bunch->Add(blob_tail), - "Blob file with overlapping row id should have decreasing sequence number."); + // Files sharing a max sequence number form one layer and must not overlap. + ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), + "Blob files with the same max sequence number should not have overlapping " + "row id ranges"); } TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithNonContinuousRowId) { @@ -265,12 +281,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, blob_entry is removed + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is a newer + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_sub2); - ASSERT_EQ(blob_bunch->RowCount(), 5); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); + ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { @@ -291,13 +309,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, as blob_sub2 has smaller sequence number, blob_sub2 will be - // skipped. + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is an older + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); + ASSERT_EQ(blob_bunch->Files().size(), 2); ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelection) { @@ -420,15 +439,15 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { std::vector> batch = batches[0]; ASSERT_EQ(batch.size(), 10); ASSERT_EQ(batch[0], data); - ASSERT_EQ(batch[1], blob_entry5); // pick - ASSERT_EQ(batch[2], blob_entry2); // skip - ASSERT_EQ(batch[3], blob_entry1); // skip - ASSERT_EQ(batch[4], blob_entry9); // pick - ASSERT_EQ(batch[5], blob_entry6); // skip - ASSERT_EQ(batch[6], blob_entry3); // skip - ASSERT_EQ(batch[7], blob_entry7); // pick - ASSERT_EQ(batch[8], blob_entry4); // skip - ASSERT_EQ(batch[9], blob_entry8); // pick + ASSERT_EQ(batch[1], blob_entry5); + ASSERT_EQ(batch[2], blob_entry2); + ASSERT_EQ(batch[3], blob_entry1); + ASSERT_EQ(batch[4], blob_entry9); + ASSERT_EQ(batch[5], blob_entry6); + ASSERT_EQ(batch[6], blob_entry3); + ASSERT_EQ(batch[7], blob_entry7); + ASSERT_EQ(batch[8], blob_entry4); + ASSERT_EQ(batch[9], blob_entry8); auto blob_field_to_field_id = [](const std::shared_ptr&) -> Result { return 0; @@ -440,12 +459,19 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { ASSERT_EQ(bunch.size(), 2); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { @@ -561,20 +587,33 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { ASSERT_EQ(bunch.size(), 3); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); auto blob_bunch2 = std::dynamic_pointer_cast(bunch[2]); - ASSERT_EQ(blob_bunch2->Files().size(), 4); + ASSERT_EQ(blob_bunch2->Files().size(), 9); ASSERT_EQ(blob_bunch2->Files()[0], blob_entry15); - ASSERT_EQ(blob_bunch2->Files()[1], blob_entry19); - ASSERT_EQ(blob_bunch2->Files()[2], blob_entry17); - ASSERT_EQ(blob_bunch2->Files()[3], blob_entry18); + ASSERT_EQ(blob_bunch2->Files()[1], blob_entry12); + ASSERT_EQ(blob_bunch2->Files()[2], blob_entry11); + ASSERT_EQ(blob_bunch2->Files()[3], blob_entry19); + ASSERT_EQ(blob_bunch2->Files()[4], blob_entry16); + ASSERT_EQ(blob_bunch2->Files()[5], blob_entry13); + ASSERT_EQ(blob_bunch2->Files()[6], blob_entry17); + ASSERT_EQ(blob_bunch2->Files()[7], blob_entry14); + ASSERT_EQ(blob_bunch2->Files()[8], blob_entry18); ASSERT_EQ(blob_bunch2->RowCount(), 1000); + ASSERT_FALSE(blob_bunch2->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestDifferentRowIdRange) { diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index 86df4841d..e1cee5a08 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -272,7 +272,8 @@ Result> MergeFileSplitRead::CreateNoMergeReader( std::vector> raw_file_readers, CreateRawFileReaders(data_split->Partition(), data_split->DataFiles(), read_schema, only_filter_key ? predicate_for_keys_ : context_->GetPredicate(), - dv_factory, /*row_ranges=*/{}, data_file_path_factory)); + dv_factory, /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -496,7 +497,8 @@ Result> MergeFileSplitRead::CreateReaderFo PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); assert(data_files.size() == raw_file_readers.size()); // KeyValueDataFileRecordReader converts arrow array from format reader to KeyValue objects diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index ff213a4ea..c027e1c72 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -81,7 +81,8 @@ Result> RawFileSplitRead::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.cpp b/src/paimon/core/table/source/data_evolution_batch_scan.cpp index 94310b2e5..cabb03d85 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan.cpp +++ b/src/paimon/core/table/source/data_evolution_batch_scan.cpp @@ -16,6 +16,9 @@ #include "paimon/core/table/source/data_evolution_batch_scan.h" +#include +#include + #include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/table/source/data_split_impl.h" @@ -72,7 +75,7 @@ Result> DataEvolutionBatchScan::CreatePlan() { Result> DataEvolutionBatchScan::WrapToIndexedSplits( const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index, - const std::map& id_to_score) const { + const std::map& id_to_score) { // TODO(lisizhuo.lsz): add executor here auto data_splits = data_plan->Splits(); std::vector> indexed_splits; @@ -86,11 +89,23 @@ Result> DataEvolutionBatchScan::WrapToIndexedSplits( if (files.empty()) { return Status::Invalid("Empty data files in WrapToIndexedSplits"); } - PAIMON_ASSIGN_OR_RAISE(int64_t min, files[0]->NonNullFirstRowId()); - PAIMON_ASSIGN_OR_RAISE(int64_t max, files[files.size() - 1]->NonNullFirstRowId()); - max += files[files.size() - 1]->row_count - 1; + // The row-id ranges of the files in a split may be unordered, discontiguous, or + // overlapping, so intersect the index with each file's range separately, then sort and + // merge the intersected ranges. + std::vector intersected; + int64_t min = std::numeric_limits::max(); + int64_t max = std::numeric_limits::min(); + for (const auto& file : files) { + PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); + int64_t last_row_id = first_row_id + file->row_count - 1; + min = std::min(min, first_row_id); + max = std::max(max, last_row_id); + std::vector file_ranges = + row_range_index.IntersectedRanges(first_row_id, last_row_id); + intersected.insert(intersected.end(), file_ranges.begin(), file_ranges.end()); + } - std::vector expected = row_range_index.IntersectedRanges(min, max); + std::vector expected = Range::SortAndMergeOverlap(intersected, /*adjacent=*/true); if (expected.empty()) { return Status::Invalid( fmt::format("There should be intersected ranges for split with min row id {} and " diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.h b/src/paimon/core/table/source/data_evolution_batch_scan.h index 88f20f762..306a25159 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan.h +++ b/src/paimon/core/table/source/data_evolution_batch_scan.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -37,10 +38,14 @@ class DataEvolutionBatchScan : public AbstractTableScan { Result> CreatePlan() override; - private: - Result> WrapToIndexedSplits( + /// Wraps each DataSplit in `data_plan` into an IndexedSplit whose row ranges are the + /// intersection of `row_range_index` and the row-id range of each data file. Visible for + /// testing. + static Result> WrapToIndexedSplits( const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index, - const std::map& id_to_score) const; + const std::map& id_to_score); + + private: Result> EvalGlobalIndex() const; private: diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp new file mode 100644 index 000000000..9081d7fd0 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp @@ -0,0 +1,113 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/source/data_evolution_batch_scan.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/plan_impl.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon::test { +namespace { +std::shared_ptr NewAppendFile(const std::string& file_name, int64_t first_row_id, + int64_t row_count) { + return std::make_shared( + file_name, /*file_size=*/1024l, row_count, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_sequence_number=*/0l, + /*max_sequence_number=*/first_row_id + row_count - 1, /*schema_id=*/0, /*level=*/0, + std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0, + /*embedded_index=*/nullptr, FileSource::Append(), std::nullopt, std::nullopt, first_row_id, + std::nullopt); +} + +std::shared_ptr NewDataPlan(std::vector> files) { + DataSplitImpl::Builder builder( + /*partition=*/BinaryRow::EmptyRow(), + /*bucket=*/0, /*bucket_path=*/"data/test_table/bucket-0", std::move(files)); + std::shared_ptr data_split = + builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value(); + return std::make_shared(/*snapshot_id=*/1, + std::vector>({data_split})); +} +} // namespace + +TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithUnorderedAndDiscontiguousDataFiles) { + // The files cover [4650, 4700], [4300, 4450] and [4200, 4407]: unordered, partially + // overlapping, with a gap [4451, 4649] that must not appear in the wrapped ranges. + std::shared_ptr data_plan = + NewDataPlan({NewAppendFile("file-1", 4650l, 51l), NewAppendFile("file-2", 4300l, 151l), + NewAppendFile("file-3", 4200l, 208l)}); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 5000)})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, + DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index, + /*id_to_score=*/{})); + + ASSERT_EQ(plan->Splits().size(), 1); + auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_NE(indexed_split, nullptr); + ASSERT_EQ(indexed_split->GetDataSplit(), data_plan->Splits()[0]); + ASSERT_EQ(indexed_split->RowRanges(), + std::vector({Range(4200, 4450), Range(4650, 4700)})); + ASSERT_TRUE(indexed_split->Scores().empty()); +} + +TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsExcludesRowIdsInFileRangeGaps) { + // The split covers [0, 9] and [20, 29] while the index hits {5, 15, 25}. Row id 15 lies in + // the gap between the two files and must be excluded, together with its score. + std::shared_ptr data_plan = + NewDataPlan({NewAppendFile("file-1", 0l, 10l), NewAppendFile("file-2", 20l, 10l)}); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, + RowRangeIndex::Create({Range(5, 5), Range(15, 15), Range(25, 25)})); + std::map id_to_score = {{5l, 0.5f}, {15l, 0.7f}, {25l, 0.9f}}; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, DataEvolutionBatchScan::WrapToIndexedSplits( + data_plan, row_range_index, id_to_score)); + + ASSERT_EQ(plan->Splits().size(), 1); + auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_NE(indexed_split, nullptr); + ASSERT_EQ(indexed_split->RowRanges(), std::vector({Range(5, 5), Range(25, 25)})); + ASSERT_EQ(indexed_split->Scores(), std::vector({0.5f, 0.9f})); +} + +TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithoutIntersection) { + std::shared_ptr data_plan = NewDataPlan({NewAppendFile("file-1", 100l, 10l)}); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 50)})); + + ASSERT_NOK_WITH_MSG(DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index, + /*id_to_score=*/{}), + "There should be intersected ranges for split with min row id 100 and " + "max row id 109."); +} + +} // namespace paimon::test diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index d5b5a2fe2..43156fe3e 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -17,6 +17,7 @@ #include "paimon/format/blob/blob_file_batch_reader.h" #include +#include #include #include "arrow/api.h" @@ -38,7 +39,7 @@ namespace paimon::blob { Result> BlobFileBatchReader::Create( const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor, - const std::shared_ptr& pool) { + bool emit_placeholder_sentinel, const std::shared_ptr& pool) { if (input_stream == nullptr) { return Status::Invalid("blob file batch reader create failed: input stream is nullptr"); } @@ -83,14 +84,15 @@ Result> BlobFileBatchReader::Create( int64_t offset = 0; for (const auto& blob_length : blob_lengths) { blob_offsets.push_back(offset); - // Null blobs (bin_length == -1) don't occupy file space + // null (-1) and placeholder (-2) entries occupy no file space if (blob_length >= 0) { offset += blob_length; } } PAIMON_ASSIGN_OR_RAISE(std::string file_path, input_stream->GetUri()); - auto reader = std::unique_ptr(new BlobFileBatchReader( - input_stream, file_path, blob_lengths, blob_offsets, batch_size, blob_as_descriptor, pool)); + auto reader = std::unique_ptr( + new BlobFileBatchReader(input_stream, file_path, blob_lengths, blob_offsets, batch_size, + blob_as_descriptor, emit_placeholder_sentinel, pool)); return reader; } @@ -99,6 +101,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, bool blob_as_descriptor, + bool emit_placeholder_sentinel, const std::shared_ptr& pool) : input_stream_(input_stream), file_path_(file_path), @@ -108,6 +111,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp target_blob_offsets_(blob_offsets), batch_size_(batch_size), blob_as_descriptor_(blob_as_descriptor), + emit_placeholder_sentinel_(emit_placeholder_sentinel), pool_(pool), arrow_pool_(GetArrowPool(pool_)), metrics_(std::make_shared()) { @@ -156,7 +160,7 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, } target_type_ = arrow::struct_(arrow_schema->fields()); current_pos_ = 0; - previous_batch_first_row_number_ = std::numeric_limits::max(); + previous_batch_start_pos_ = std::numeric_limits::max(); previous_batch_row_count_ = 0; return Status::OK(); } @@ -168,11 +172,7 @@ Result> BlobFileBatchReader::NextBlobOffsets( PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(0)); int64_t data_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - // Null blobs contribute zero bytes to content - if (!IsTargetNull(i)) { - data_length += GetTargetContentLength(i); - } + data_length += GetTargetOutputLength(current_pos_ + k); PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(data_length)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr offset_buffer, @@ -184,10 +184,7 @@ Result> BlobFileBatchReader::NextBlobContents( int32_t rows_to_read) const { int64_t total_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - if (!IsTargetNull(i)) { - total_length += GetTargetContentLength(i); - } + total_length += GetTargetOutputLength(current_pos_ + k); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr data_buffer, arrow::AllocateBuffer(total_length, arrow_pool_.get())); @@ -197,6 +194,13 @@ Result> BlobFileBatchReader::NextBlobContents( if (IsTargetNull(i)) { continue; } + if (IsTargetPlaceholder(i)) { + // a placeholder entry has no data bytes in the file; emit the sentinel for the + // data-evolution blob fallback merge to identify it + memcpy(buffer, BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength); + buffer += BlobDefs::kPlaceholderSentinelLength; + continue; + } int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); PAIMON_RETURN_NOT_OK(ReadBlobContentAt(offset, length, buffer)); @@ -268,6 +272,9 @@ Result> BlobFileBatchReader::BuildTargetArray( PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append()); if (IsTargetNull(i)) { PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull()); + } else if (IsTargetPlaceholder(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append( + BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength)); } else { int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); @@ -291,18 +298,29 @@ Result BlobFileBatchReader::NextBatch() { return Status::Invalid("target type is nullptr, call SetReadSchema first"); } if (current_pos_ >= target_blob_lengths_.size()) { - PAIMON_ASSIGN_OR_RAISE(previous_batch_first_row_number_, GetNumberOfRows()); + previous_batch_start_pos_ = target_blob_lengths_.size(); previous_batch_row_count_ = 0; return BatchReader::MakeEofBatch(); } int32_t left_rows = target_blob_lengths_.size() - current_pos_; int32_t rows_to_read = std::min(left_rows, batch_size_); + if (!emit_placeholder_sentinel_) { + for (int32_t k = 0; k < rows_to_read; ++k) { + if (IsTargetPlaceholder(current_pos_ + k)) { + return Status::Invalid(fmt::format( + "blob file {} contains a placeholder entry (bin_length {}) written by a " + "data-evolution partial update; it can only be resolved by the data-evolution " + "blob fallback read path", + file_path_, BlobDefs::kPlaceholderBinLength)); + } + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob_array, BuildTargetArray(rows_to_read)); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*blob_array, c_array.get(), c_schema.get())); - previous_batch_first_row_number_ = target_blob_row_indexes_[current_pos_]; + previous_batch_start_pos_ = current_pos_; current_pos_ += rows_to_read; previous_batch_row_count_ = c_array->length; return make_pair(std::move(c_array), std::move(c_schema)); diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 27549c190..7747b60c8 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -49,7 +49,8 @@ namespace paimon::blob { /// ==================================================================== /// 1. Data Bins Section /// ==================================================================== -/// The file consists of one or more contiguous 'bins' (bin_0, bin_1, bin_2, ...). +/// The file consists of zero or more contiguous 'bins' (bin_0, bin_1, bin_2, ...); rows whose +/// index entry is negative (see Section 2) have no bin. /// The structure of each bin is as follows: /// /// | Field Name | Length (bytes) | Description | @@ -67,7 +68,13 @@ namespace paimon::blob { /// ==================================================================== /// The Index is located after all data bins and is used for quick lookup and management. /// -/// Purpose: Records the lengths (record lens) of all data bins. +/// Purpose: Records one signed length entry per row, in row order. +/// +/// Special lengths: an entry does not always describe a bin present in the Data Bins Section. +/// - -1 (BlobDefs::kNullBinLength): a null blob; no bin is written. +/// - -2 (BlobDefs::kPlaceholderBinLength): a placeholder blob written by a data-evolution +/// partial update for a row it did not touch; no bin is written and the value must be +/// resolved from an older blob file covering the same row. /// /// Encoding: /// - Uses Delta Encoding to store differences between successive length values. @@ -87,9 +94,16 @@ namespace paimon::blob { /// - Current version is 1. class BlobFileBatchReader : public FileBatchReader { public: + /// `emit_placeholder_sentinel` controls how placeholder entries (bin_length == + /// BlobDefs::kPlaceholderBinLength) are read: when false they fail the read, as resolving + /// them requires the data-evolution blob fallback path; when true they are returned as the + /// non-null BlobDefs::kPlaceholderSentinel bytes for that path to merge away. Stored values + /// are returned verbatim; see BlobDefs::kPlaceholderSentinel for the accepted collision + /// with a user value exactly equal to the sentinel. static Result> Create( const std::shared_ptr& input_stream, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Result> GetFileSchema() const override; @@ -99,14 +113,8 @@ class BlobFileBatchReader : public FileBatchReader { Result NextBatch() override; Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { - if (all_blob_lengths_.size() != target_blob_lengths_.size()) { - return Status::NotImplemented( - "Cannot call GetPreviousBatchFileRowId in BlobFileBatchReader because, after " - "bitmap pushdown, rows in the array returned by NextBatch are no longer " - "contiguous."); - } if (previous_batch_row_count_ == 0) { - if (previous_batch_first_row_number_ == std::numeric_limits::max()) { + if (previous_batch_start_pos_ == std::numeric_limits::max()) { return Status::Invalid("No batch has been read yet."); } else { return Status::Invalid("Last batch was EOF."); @@ -117,7 +125,9 @@ class BlobFileBatchReader : public FileBatchReader { fmt::format("batch_row_id {} is out of range, last batch row count is {}", batch_row_id, previous_batch_row_count_)); } - return previous_batch_first_row_number_ + batch_row_id; + // target_blob_row_indexes_ maps every selected position back to its original file row + // index, so this stays correct after a bitmap selection removed rows. + return target_blob_row_indexes_[previous_batch_start_pos_ + batch_row_id]; } Result GetNumberOfRows() const override { @@ -144,7 +154,8 @@ class BlobFileBatchReader : public FileBatchReader { BlobFileBatchReader(const std::shared_ptr& input_stream, const std::string& file_path, const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Status ReadBlobContentAt(const int64_t offset, const int64_t length, uint8_t* content) const; @@ -160,6 +171,23 @@ class BlobFileBatchReader : public FileBatchReader { return target_blob_lengths_[index] == BlobDefs::kNullBinLength; } + bool IsTargetPlaceholder(size_t index) const { + return target_blob_lengths_[index] == BlobDefs::kPlaceholderBinLength; + } + + /// Content bytes the blob at the given index contributes to the output buffer: nothing for + /// a null entry, the sentinel bytes for a placeholder entry (which occupies no file space), + /// the stored bytes otherwise. + int64_t GetTargetOutputLength(size_t index) const { + if (IsTargetNull(index)) { + return 0; + } + if (IsTargetPlaceholder(index)) { + return BlobDefs::kPlaceholderSentinelLength; + } + return GetTargetContentLength(index); + } + int64_t GetTargetContentOffset(size_t index) const { return target_blob_offsets_[index] + BlobDefs::kContentStartOffset; } @@ -179,6 +207,7 @@ class BlobFileBatchReader : public FileBatchReader { const int32_t batch_size_; const bool blob_as_descriptor_; + const bool emit_placeholder_sentinel_; std::shared_ptr pool_; std::shared_ptr arrow_pool_; @@ -186,7 +215,9 @@ class BlobFileBatchReader : public FileBatchReader { std::shared_ptr metrics_; size_t current_pos_ = 0; - uint64_t previous_batch_first_row_number_ = std::numeric_limits::max(); + /// Start position of the previous batch in the (possibly selection-filtered) target index + /// space; max() until the first batch is read. + size_t previous_batch_start_pos_ = std::numeric_limits::max(); uint64_t previous_batch_row_count_ = 0; bool closed_ = false; }; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 4e53dfb4c..866e136c2 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -44,9 +44,9 @@ class BlobFileBatchReaderTest : public testing::Test, public ::testing::WithPara std::shared_ptr fs = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/" + paimon_blob_file)); - ASSERT_OK_AND_ASSIGN(auto reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, - blob_as_descriptor, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( + input_stream, /*batch_size=*/1024, blob_as_descriptor, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection_bitmap)); ASSERT_OK_AND_ASSIGN(auto chunked_array, paimon::test::ReadResultCollector::CollectResult(reader.get())); @@ -162,9 +162,10 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_OK_AND_ASSIGN( std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -187,6 +188,45 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); } +TEST_F(BlobFileBatchReaderTest, TestRowNumbersWithSelectionBitmap) { + // a selection bitmap removes rows, but batch positions must still map back to the original + // file row indexes so _ROW_ID completion works under row-range pushdown + auto schema = arrow::schema({BlobUtils::ToArrowField("my_blob_field", false)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + + std::string test_data_path = paimon::test::GetDataDir() + "/db_with_blob.db/table_with_blob/"; + auto dir = paimon::test::UniqueTestDirectory::Create(); + std::string table_path = dir->Str(); + ASSERT_TRUE(paimon::test::TestUtil::CopyDirectory(test_data_path, table_path)); + + std::shared_ptr fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr input_stream, + fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); + + RoaringBitmap32 selection; + selection.Add(0); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection)); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "No batch has been read yet"); + ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch()); + ArrowArrayRelease(batch1.first.get()); + ArrowSchemaRelease(batch1.second.get()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); + ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch()); + ArrowArrayRelease(batch2.first.get()); + ArrowSchemaRelease(batch2.second.get()); + ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value()); + ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch()); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "Last batch was EOF"); + ASSERT_TRUE(BatchReader::IsEofBatch(batch3)); +} + TEST_F(BlobFileBatchReaderTest, InvalidScenario) { auto dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -202,20 +242,22 @@ TEST_F(BlobFileBatchReaderTest, InvalidScenario) { { ASSERT_NOK_WITH_MSG( BlobFileBatchReader::Create(input_stream, - /*batch_size=*/0, /*blob_as_descriptor=*/true, pool_), + /*batch_size=*/0, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_), "blob file batch reader create failed: read batch size '0' should be larger than zero"); } { ASSERT_NOK_WITH_MSG( BlobFileBatchReader::Create(/*input_stream=*/nullptr, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_), + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_), "blob file batch reader create failed: input stream is nullptr"); } { ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(/*input_stream=*/input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(/*input_stream=*/input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->GetFileSchema(), "blob file has no self-describing file schema"); ASSERT_TRUE(reader->GetReaderMetrics()); @@ -237,7 +279,8 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream, struct_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -248,9 +291,10 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(dir->Str() + "/file.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -273,9 +317,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(/*read_schema=*/nullptr, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "SetReadSchema failed: read schema cannot be nullptr"); @@ -298,9 +342,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "read schema field number 2 is not 1"); @@ -323,9 +367,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "field my_blob_field: large_binary is not BLOB"); @@ -346,9 +390,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); RoaringBitmap32 roaring; roaring.Add(0); roaring.Add(1); diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index 875566ffc..b98e56c55 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -39,7 +39,7 @@ namespace paimon::blob { BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, - bool write_null_on_fetch_failure, + bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) : out_(out), @@ -48,7 +48,8 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con fs_(fs), pool_(pool), write_null_on_missing_file_(write_null_on_missing_file), - write_null_on_fetch_failure_(write_null_on_fetch_failure) { + write_null_on_fetch_failure_(write_null_on_fetch_failure), + write_placeholder_(write_placeholder) { // Create() has already checked that data_type has exactly one BLOB field. blob_field_name_ = data_type_->field(0)->name(); metrics_ = std::make_shared(); @@ -59,7 +60,7 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con Result> BlobFormatWriter::Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); @@ -82,8 +83,9 @@ Result> BlobFormatWriter::Create( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); - return std::unique_ptr(new BlobFormatWriter( - out, uri, data_type, write_null_on_missing_file, write_null_on_fetch_failure, fs, pool)); + return std::unique_ptr( + new BlobFormatWriter(out, uri, data_type, write_null_on_missing_file, + write_null_on_fetch_failure, write_placeholder, fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -117,7 +119,16 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { const auto& blob_array = arrow::internal::checked_cast(*child_array); assert(blob_array.length() == 1); - PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0))); + std::string_view blob_data = blob_array.GetView(0); + // Only a data-evolution partial-update write interprets the sentinel, which marks a row + // the update did not touch; any other write stores the bytes verbatim. See + // BlobDefs::kPlaceholderSentinel for the accepted collision with a sentinel-equal user + // value. + if (write_placeholder_ && BlobDefs::IsPlaceholderSentinel(blob_data.data(), blob_data.size())) { + bin_lengths_.push_back(BlobDefs::kPlaceholderBinLength); + return Status::OK(); + } + PAIMON_RETURN_NOT_OK(WriteBlob(blob_data)); PAIMON_RETURN_NOT_OK(Flush()); return Status::OK(); } diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 9dbe57a87..6111168dc 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -68,9 +68,15 @@ class BlobFormatWriter : public FormatWriter { /// `write_null_on_missing_file` is enabled; otherwise a missing file follows /// `write_null_on_fetch_failure` like any other failed open. /// See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. + /// + /// `write_placeholder` (see BlobDefs::kWritePlaceholderKey, false unless the write is a + /// data-evolution partial update) persists a value exactly equal to + /// BlobDefs::kPlaceholderSentinel as a placeholder entry (bin_length -2, no data bytes); + /// any other value is stored verbatim. When disabled, values are never interpreted and can + /// never be turned into placeholder entries. static Result> Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -91,7 +97,7 @@ class BlobFormatWriter : public FormatWriter { BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, bool write_null_on_fetch_failure, - const std::shared_ptr& fs, + bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); @@ -135,6 +141,7 @@ class BlobFormatWriter : public FormatWriter { std::shared_ptr metrics_; bool write_null_on_missing_file_ = false; bool write_null_on_fetch_failure_ = false; + bool write_placeholder_ = false; uint64_t null_on_missing_file_count_ = 0; uint64_t null_on_fetch_failure_count_ = 0; std::unique_ptr logger_; diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index d54f6042c..74b5b3b6c 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -129,7 +129,16 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> CreateDefaultWriter() const { return BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_); + } + + /// Create a writer in placeholder mode, as used by data-evolution partial updates. + Result> CreatePlaceholderWriter() const { + return BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/true, file_system_, pool_); } Status AddBatchOnce(const std::shared_ptr& format_writer, @@ -144,8 +153,7 @@ class BlobFormatWriterTestBase : public ::testing::Test { return paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, pool_); } - /// Build a single-row blob array holding `bytes` verbatim, for bytes no Blob can produce, - /// such as a truncated descriptor. + /// Build a single-row blob array holding `bytes` verbatim, bypassing the Blob helpers. Result> MakeBlobArrayFromBytes(const std::string& bytes) const { arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), {std::make_shared()}); @@ -161,9 +169,11 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> ReadBackAsData() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, - /*blob_as_descriptor=*/false, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); @@ -253,7 +263,8 @@ TEST_P(BlobFormatWriterTest, TestSimple) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -284,41 +295,47 @@ TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(nullptr, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. out is nullptr"); // Test with nullptr data type ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, nullptr, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, nullptr), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, nullptr), "blob format writer create failed. pool is nullptr"); // Test with nullptr file system ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, nullptr, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, nullptr, pool_), "blob format writer create failed. fs is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, multi_field_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, non_blob_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "field regular_col: binary is not BLOB"); } @@ -443,7 +460,8 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -492,7 +510,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -525,7 +544,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, @@ -563,7 +583,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); std::string file = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, @@ -602,7 +623,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); // Row 0: missing file -> NULL. ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, @@ -667,7 +689,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), 0); } @@ -676,7 +699,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); } // The same fetch failure, now converted to NULL. @@ -684,7 +708,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); } // Missing file with only fetch-failure enabled: no existence check runs, so the file is @@ -695,7 +720,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), open_calls_before + 1); ASSERT_OK_AND_ASSIGN( @@ -713,7 +739,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); // One check before the open and one after it. ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); @@ -733,7 +760,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); ASSERT_OK_AND_ASSIGN( @@ -752,7 +780,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -762,7 +791,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -785,14 +815,16 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnInvalidDescriptor) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "invalid blob descriptor"); } { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -820,7 +852,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 1); ASSERT_OK(writer->Flush()); @@ -850,7 +883,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, exists_fail_fs, pool_)); // The reported failure names the check and keeps the underlying status message. Status check_status = AddBatchOnce(writer, array); ASSERT_NOK_WITH_MSG(check_status, "failed to check existence of blob file"); @@ -864,7 +898,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); // One check before the open and one after it failed. ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 2); @@ -884,7 +919,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( side_stream, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 0); ASSERT_OK_AND_ASSIGN( @@ -931,4 +967,238 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_EQ(buffer, expected); } +/// Placeholder tests always feed the sentinel bytes of the placeholder write protocol, so +/// they do not depend on the blob_as_descriptor_ parameter and run once on the +/// non-parameterized fixture. +using BlobFormatWriterPlaceholderTest = BlobFormatWriterTestBase; + +std::string PlaceholderSentinelBytes() { + return std::string(BlobDefs::PlaceholderSentinelView()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestWritePlaceholderGoldenBytes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + + // row 0: inline bytes "inline"; row 1: null; row 2: placeholder + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(blob_builder->AppendNull().ok()); + std::shared_ptr null_array; + ASSERT_TRUE(struct_builder.Finish(&null_array).ok()); + ASSERT_OK(AddBatchOnce(writer, null_array)); + + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + // Verify byte-level alignment with the Java writer (BlobFormatWriterTest + // testRawBlobGoldenBytes): null and placeholder rows occupy no data bytes; the index + // records [22, -1, -2] as zigzag varint deltas [0x2c, 0x2d, 0x01]. + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_TRUE(input_stream); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); + ASSERT_EQ(file_length, 30); + std::vector buffer(file_length); + ASSERT_OK_AND_ASSIGN(auto read_length, + input_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); + ASSERT_EQ(read_length, 30); + std::vector expected = {{// record 0: magic + "inline" + bin_length(22) + crc32 + 0xcf, 0x11, 0x4e, 0x58, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x60, + 0xc8, 0xe9, + // index of [22, -1, -2] + 0x2c, 0x2d, 0x01, + // footer: index length + version + 0x03, 0x00, 0x00, 0x00, 0x01}}; + ASSERT_EQ(buffer, expected); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderStrictAndAwareModes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + + // default (strict) mode: reading a placeholder entry fails + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "placeholder"); + } + + // placeholder-aware mode: the entry is returned as the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), "inline"); + ASSERT_FALSE(binary_array->IsNull(1)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } + + // placeholder-aware descriptor mode also returns the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderWithSelectionBitmap) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto array0, MakeBlobArrayFromBytes("first")); + ASSERT_OK(AddBatchOnce(writer, array0)); + ASSERT_OK_AND_ASSIGN(auto array1, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, array1)); + ASSERT_OK_AND_ASSIGN(auto array2, MakeBlobArrayFromBytes("third")); + ASSERT_OK(AddBatchOnce(writer, array2)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + RoaringBitmap32 selection; + selection.Add(1); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, selection)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); + ASSERT_EQ(binary_array->GetString(1), "third"); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelBytesVerbatimWithoutPlaceholderMode) { + // outside placeholder mode a user blob whose bytes equal the sentinel is a normal value: it + // must be stored as a real entry (not persisted as bin_length -2) and read back unchanged + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); + ASSERT_OK_AND_ASSIGN(auto sentinel_array, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, sentinel_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 1); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_FALSE(binary_array->IsNull(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelPrefixedValueVerbatimInPlaceholderMode) { + // placeholders are identified by exact equality only: even in placeholder mode a real + // value that merely starts with the sentinel bytes is stored verbatim and read back + // unchanged in both strict and placeholder-aware modes + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + std::string sentinel = PlaceholderSentinelBytes(); + ASSERT_OK_AND_ASSIGN(auto doubled_sentinel_array, MakeBlobArrayFromBytes(sentinel + sentinel)); + ASSERT_OK(AddBatchOnce(writer, doubled_sentinel_array)); + ASSERT_OK_AND_ASSIGN(auto prefixed_array, MakeBlobArrayFromBytes(sentinel + "suffix")); + ASSERT_OK(AddBatchOnce(writer, prefixed_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + for (bool emit_placeholder_sentinel : {false, true}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + emit_placeholder_sentinel, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(struct_array->length(), 2); + ASSERT_EQ(binary_array->GetString(0), sentinel + sentinel); + ASSERT_EQ(binary_array->GetString(1), sentinel + "suffix"); + } +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/blob/blob_reader_builder.h b/src/paimon/format/blob/blob_reader_builder.h index eb2ce9522..a5d20fbdd 100644 --- a/src/paimon/format/blob/blob_reader_builder.h +++ b/src/paimon/format/blob/blob_reader_builder.h @@ -43,7 +43,11 @@ class BlobReaderBuilder : public ReaderBuilder { PAIMON_ASSIGN_OR_RAISE( bool blob_as_descriptor, OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false)); - return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, pool_); + PAIMON_ASSIGN_OR_RAISE(bool emit_placeholder_sentinel, + OptionsUtils::GetValueFromMap( + options_, BlobDefs::kEmitPlaceholderSentinelKey, false)); + return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, + emit_placeholder_sentinel, pool_); } private: diff --git a/src/paimon/format/blob/blob_stats_extractor.cpp b/src/paimon/format/blob/blob_stats_extractor.cpp index ca5efe831..e840012e7 100644 --- a/src/paimon/format/blob/blob_stats_extractor.cpp +++ b/src/paimon/format/blob/blob_stats_extractor.cpp @@ -48,10 +48,13 @@ BlobStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_ fmt::format("field {} is not BLOB", write_schema_->field(0)->ToString())); } + // The reader only serves footer metadata (GetNumberOfRows); NextBatch is never called, so + // the strict placeholder mode is irrelevant even for files containing placeholder entries. PAIMON_ASSIGN_OR_RAISE( std::unique_ptr blob_reader, BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1024, /*blob_as_descriptor=*/true, pool)); + /*batch_size=*/1024, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool)); ColumnStatsVector result_stats; result_stats.push_back( ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt)); diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index 12caa0e16..7cc0b5927 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -24,6 +24,7 @@ #include #include "arrow/api.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/utils/options_utils.h" #include "paimon/defs.h" #include "paimon/format/blob/blob_format_writer.h" @@ -73,8 +74,11 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { PAIMON_ASSIGN_OR_RAISE(bool write_null_on_fetch_failure, OptionsUtils::GetValueFromMap( options_, Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, false)); + PAIMON_ASSIGN_OR_RAISE( + bool write_placeholder, + OptionsUtils::GetValueFromMap(options_, BlobDefs::kWritePlaceholderKey, false)); return BlobFormatWriter::Create(out, data_type_, write_null_on_missing_file, - write_null_on_fetch_failure, fs_, pool_); + write_null_on_fetch_failure, write_placeholder, fs_, pool_); } private: diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 398482dae..560f77217 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -42,6 +42,7 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" @@ -1101,6 +1102,595 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) { ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); } +/// Build a single-blob-column StructArray for a data-evolution partial update: "PH" marks a row +/// whose blob is not updated (persisted as a placeholder entry), std::nullopt a null blob. +std::shared_ptr MakeBlobUpdateArray( + const std::shared_ptr& blob_field, + const std::vector>& rows) { + auto struct_type = arrow::struct_({blob_field}); + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string_view sentinel = BlobDefs::PlaceholderSentinelView(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return std::dynamic_pointer_cast(array); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { + // the blob column is updated to null below, so it must be nullable + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Initial full-row write assigns row ids 0-2. + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "blob_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // Partial update: only row 1 gets a new blob, the untouched rows are written as + // placeholder entries and must fall back to the previous blob file when read. + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // updating a blob to null is not a placeholder: the null must win over older layers + auto null_update_array = MakeBlobUpdateArray(fields[2], {std::nullopt, "PH", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {}, {"b0"}, {null_update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array2)); + + // row ids still come from the data files and stay aligned with the fallback result + auto expected_with_row_id = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}), + R"([ + [1, "a", null, 0], + [2, "b", "updated_b", 1], + [3, "c", "blob_c", 2] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); + + // blob_as_descriptor read mode: the fallback-merged values come back as descriptors and + // must still resolve to the same bytes + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto desc_result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(desc_result.chunked_array); + auto desc_concat = arrow::Concatenate(desc_result.chunked_array->chunks()).ValueOrDie(); + auto desc_struct = std::dynamic_pointer_cast(desc_concat); + ASSERT_TRUE(desc_struct); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(desc_struct, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array2)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << "\nexpected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "blob_3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // second layer updates row 0 within rows [0, 1] + auto update_array1 = MakeBlobUpdateArray(fields[2], {"update1_0", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // third layer updates row 3 within rows [2, 3]; each layer only partially covers the + // range, the uncovered parts behave as placeholders + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "update2_3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "update1_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "update2_3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // row-range pushdown where the selected rows fall in one layer's file and in another + // layer's uncovered gap: row 1 is a gap row for the third layer, row 2 for the second + auto expected_middle = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [2, "b", "blob_1"], + [3, "c", "blob_2"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_middle, + /*predicate=*/nullptr, /*row_ranges=*/{Range(1, 2)})); + + // per-row reads resolve every row independently + const std::vector expected_rows = { + R"([[1, "a", "update1_0"]])", R"([[2, "b", "blob_1"]])", R"([[3, "c", "blob_2"]])", + R"([[4, "d", "update2_3"]])"}; + for (int32_t i = 0; i < static_cast(expected_rows.size()); i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_rows[i]) + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Mirrors the layer shapes of Java's BlobUpdateTest.testReadCompactedBlobSequenceGroups + // (with a single base write, as the C++ write path assigns one sequence per commit): + // row id: 0 1 2 3 4 5 6 7 8 9 + // seq1: [b0 b1 b2 b3 b4 b5 b6 b7 b8 b9] + // seq2: [u20 P P u23 u24] . . . . . + // seq3: . . . . . [P u46 P u48 P] + // seq4: [P u61 P P P P P P P u69] + // result: u20 u61 b2 u23 u24 b5 u46 b7 u48 u69 + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array1 = MakeBlobUpdateArray(fields[2], {"u20", "PH", "PH", "u23", "u24"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "u46", "PH", "u48", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/5, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto update_array3 = MakeBlobUpdateArray( + fields[2], {"PH", "u61", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "u69"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs3, WriteArray(table_path, {}, {"b0"}, {update_array3})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs3); + ASSERT_OK(Commit(table_path, commit_msgs3)); + + const std::vector expected_blobs = {"u20", "u61", "blob_2", "u23", "u24", + "blob_5", "u46", "blob_7", "u48", "u69"}; + auto expected_row = [&](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"" + expected_blobs[i] + + "\""; + }; + auto expected_full = PrepareBulkData(10, expected_row, fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); + + // every row resolves independently under row-range pushdown + for (int32_t i = 0; i < 10; i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + "[[" + expected_row(i) + "]]") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // partial update touching rows 1 and 9 only + auto update_array = MakeBlobUpdateArray( + fields[2], {"PH", "update_1", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "update_9"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // push down row ranges hitting updated and untouched rows + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "name_1", "update_1"], + [5, "name_5", "blob_5"], + [9, "name_9", "update_9"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 1), Range(5, 5), Range(9, 9)})); + + // full read still resolves every row + auto expected_full = PrepareBulkData( + 10, + [](int32_t i) { + std::string blob = (i == 1 || i == 9) ? "\"update_" + std::to_string(i) + "\"" + : "\"blob_" + std::to_string(i) + "\""; + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", " + blob; + }, + fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); +} + +/// A BLOB sequence layer whose physical file covers only a strict subrange of the full row id +/// range: the newest group is internally Gap(2), File([u2, u3]), Gap(6). The table needs a +/// normal column to be creatable, but only the blob column is ever written, so the split +/// contains only blob files and the blob bunch itself carries the row-tracking fields: the +/// fallback reader must keep _ROW_ID correct and report each row's _SEQUENCE_NUMBER from the +/// layer that resolved it, without ever exposing the internal placeholder sentinel. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // old layer: rows 0-9 all real + std::vector> base_rows; + for (int32_t i = 0; i < 10; i++) { + base_rows.emplace_back("b" + std::to_string(i)); + } + auto base_array = MakeBlobUpdateArray(fields[1], base_rows); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // new layer: first_row_id=2, row_count=2, covering only rows 2-3 + auto update_array = MakeBlobUpdateArray(fields[1], {"u2", "u3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 10); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + int64_t old_layer_seq = seq_col->Value(0); + int64_t new_layer_seq = seq_col->Value(2); + ASSERT_GT(new_layer_seq, old_layer_seq); + for (int64_t i = 0; i < 10; i++) { + ASSERT_FALSE(blob_col->IsNull(i)); + std::string expected_blob = + (i == 2 || i == 3) ? "u" + std::to_string(i) : "b" + std::to_string(i); + // the leading and trailing gaps never expose the internal placeholder sentinel + ASSERT_EQ(blob_col->GetString(i), expected_blob) << "row " << i; + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_EQ(seq_col->Value(i), (i == 2 || i == 3) ? new_layer_seq : old_layer_seq) + << "row " << i; + } + + // row-range pushdown with the row-tracking projection: the selection removes rows inside + // the blob files, so batch positions must still map back to the right file row indexes + ASSERT_OK_AND_ASSIGN(auto range_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"}, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 2), Range(8, 8)})); + ASSERT_TRUE(range_read.chunked_array); + auto range_concat = arrow::Concatenate(range_read.chunked_array->chunks()).ValueOrDie(); + auto range_struct = std::dynamic_pointer_cast(range_concat); + ASSERT_TRUE(range_struct); + ASSERT_EQ(range_struct->length(), 3); + auto range_blob_col = + std::dynamic_pointer_cast(range_struct->GetFieldByName("b0")); + auto range_row_id_col = std::dynamic_pointer_cast( + range_struct->GetFieldByName(SpecialFields::RowId().Name())); + auto range_seq_col = std::dynamic_pointer_cast( + range_struct->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(range_blob_col && range_row_id_col && range_seq_col); + const std::vector expected_row_ids = {1, 2, 8}; + const std::vector expected_blobs = {"b1", "u2", "b8"}; + for (int64_t i = 0; i < 3; i++) { + ASSERT_EQ(range_blob_col->GetString(i), expected_blobs[i]) << "row " << i; + ASSERT_EQ(range_row_id_col->Value(i), expected_row_ids[i]) << "row " << i; + ASSERT_EQ(range_seq_col->Value(i), expected_row_ids[i] == 2 ? new_layer_seq : old_layer_seq) + << "row " << i; + } +} + +/// A row that is a placeholder in every BLOB layer degrades to a null blob but keeps its +/// _ROW_ID and reports -1 as its _SEQUENCE_NUMBER. Only the blob column is ever written, so +/// the blob bunch itself carries the row-tracking fields. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // the base layer itself holds a placeholder at row 1, so after the second layer also + // leaves it untouched, row 1 is a placeholder in every layer + auto base_array = MakeBlobUpdateArray(fields[1], {"blob_a", "PH", "blob_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array = MakeBlobUpdateArray(fields[1], {"PH", "PH", "update_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 3); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + + ASSERT_EQ(blob_col->GetString(0), "blob_a"); + ASSERT_EQ(blob_col->GetString(2), "update_c"); + // the all-placeholder row: null blob, row id kept, sequence number -1 + ASSERT_TRUE(blob_col->IsNull(1)); + ASSERT_EQ(row_id_col->Value(1), 1); + ASSERT_EQ(seq_col->Value(1), -1); + for (int64_t i : {static_cast(0), static_cast(2)}) { + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_GE(seq_col->Value(i), 0); + } + ASSERT_GT(seq_col->Value(2), seq_col->Value(0)); +} + +/// A normal (full-row) write never interprets blob bytes: a user value whose bytes exactly +/// equal the placeholder sentinel must be stored verbatim (not persisted as a bin_length -2 +/// entry) and read back unchanged. The sentinel is reserved only inside the data-evolution +/// partial-update channels (see BlobDefs::kPlaceholderSentinel). +TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // full-row write: row 0's blob is exactly the sentinel bytes + auto struct_type = arrow::struct_(fields); + arrow::StructBuilder struct_builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(), std::make_shared(), + std::make_shared()}); + auto f0_builder = static_cast(struct_builder.field_builder(0)); + auto f1_builder = static_cast(struct_builder.field_builder(1)); + auto b0_builder = static_cast(struct_builder.field_builder(2)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(1).ok()); + ASSERT_TRUE(f1_builder->Append("a").ok()); + ASSERT_TRUE(b0_builder->Append(sentinel_bytes.data(), sentinel_bytes.size()).ok()); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(2).ok()); + ASSERT_TRUE(f1_builder->Append("b").ok()); + ASSERT_TRUE(b0_builder->Append("normal", 6).ok()); + std::shared_ptr src_array; + ASSERT_TRUE(struct_builder.Finish(&src_array).ok()); + auto src_struct = std::dynamic_pointer_cast(src_array); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_struct})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // read back unchanged: the sentinel-equal bytes are a normal value + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_struct)); + + // a value that merely starts with the sentinel bytes stays a normal value even through a + // partial-update fallback (placeholders are identified by exact equality only) + std::string prefixed_bytes = sentinel_bytes + "suffix"; + auto update_array = MakeBlobUpdateArray(fields[2], {prefixed_bytes, "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, ScanAndReadResult(table_path, schema->field_names())); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_result = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_result); + ASSERT_EQ(struct_result->length(), 2); + auto blob_col = + std::dynamic_pointer_cast(struct_result->GetFieldByName("b0")); + ASSERT_TRUE(blob_col); + ASSERT_FALSE(blob_col->IsNull(0)); + ASSERT_EQ(blob_col->GetString(0), prefixed_bytes); + ASSERT_EQ(blob_col->GetString(1), "updated_b"); +} + +TEST_P(BlobTableInteTest, TestBlobSentinelValueInBaseLayerDegradesToNull) { + // Pins the accepted collision of the byte-identified placeholder protocol (see + // BlobDefs::kPlaceholderSentinel): the fallback merge byte-compares every layer, so a user + // blob equal to the sentinel bytes that a later partial update leaves untouched reads as a + // placeholder in every layer and degrades to a null blob instead of falling back. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // the full-row write stores row 0's sentinel-equal bytes verbatim (the write channel is off) + std::string src_json = + std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])"; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // the partial update leaves row 0 untouched, so its base value collides with the + // placeholder markers of the newer layer + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // row 0's non-blob fields survive (they come from the data file); only the blob degrades + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); +} + +TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) { + // blob.internal.* options are reserved for the data-evolution write and read paths. Set in + // the user table options they must be ignored: were the write key honored, this full-row + // write would persist the sentinel-equal user value as a placeholder entry that no older + // layer can resolve, making the table unreadable. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {BlobDefs::kWritePlaceholderKey, "true"}, + {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + std::string src_json = + std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])"; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {src_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // the sentinel-equal value round-trips verbatim: the user-supplied keys were stripped + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_array)); +} + TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};