diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 07b033734..3b97dfd68 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -131,6 +131,7 @@ set(PAIMON_COMMON_SRCS common/reader/predicate_batch_reader.cpp common/reader/prefetch_file_batch_reader_impl.cpp common/reader/reader_utils.cpp + common/reader/blob_fallback_batch_reader.cpp common/reader/blob_view_resolving_batch_reader.cpp common/reader/complete_row_kind_batch_reader.cpp common/reader/data_evolution_file_reader.cpp @@ -542,6 +543,7 @@ if(PAIMON_BUILD_TESTS) common/reader/prefetch_file_batch_reader_impl_test.cpp common/reader/reader_utils_test.cpp common/reader/complete_row_kind_batch_reader_test.cpp + common/reader/blob_fallback_batch_reader_test.cpp common/reader/blob_view_resolving_batch_reader_test.cpp common/reader/data_evolution_file_reader_test.cpp common/reader/data_evolution_array_test.cpp @@ -803,6 +805,7 @@ if(PAIMON_BUILD_TESTS) core/table/source/table_read_test.cpp core/table/source/append_count_reader_test.cpp core/table/source/pk_count_reader_test.cpp + core/table/source/data_evolution_batch_scan_test.cpp core/table/source/data_split_test.cpp core/table/source/deletion_file_test.cpp core/table/source/split_generator_test.cpp diff --git a/src/paimon/common/data/blob_defs.h b/src/paimon/common/data/blob_defs.h index b8bb6f308..cd5d24323 100644 --- a/src/paimon/common/data/blob_defs.h +++ b/src/paimon/common/data/blob_defs.h @@ -17,6 +17,10 @@ #pragma once #include +#include +#include +#include +#include namespace paimon { @@ -45,6 +49,64 @@ class BlobDefs { /// A bin_length value of -1 in the index indicates a null blob entry. static constexpr int64_t kNullBinLength = -1; + /// A bin_length value of -2 in the index indicates a placeholder blob entry, written by + /// data-evolution partial updates for rows whose blob value is not updated. A placeholder + /// entry occupies no file space; readers must fall back to an older blob file covering the + /// same row to resolve the value. Aligned with Java's BlobFormatWriter.PLACE_HOLDER_LENGTH. + static constexpr int64_t kPlaceholderBinLength = -2; + /// Sentinel bytes standing for a placeholder blob value in two internal channels: + /// + /// - Write channel: a data-evolution partial update (a blob-only column write, see + /// kWritePlaceholderKey) marks a not-updated row with these bytes, and the blob format + /// writer persists it as a bin_length -2 entry. Use PlaceholderSentinelView() to build + /// such write arrays. Outside that mode the writer never interprets values, so arbitrary + /// user bytes can never be turned into a placeholder entry. + /// - Read channel: a placeholder-aware reader (see kEmitPlaceholderSentinelKey) emits these + /// bytes for -2 entries so the fallback merge can identify placeholders after the batch + /// has passed through schema-mapping readers. + /// + /// Both channels identify a placeholder by exact byte equality with this internal reserved + /// value (IsPlaceholderSentinel), and the fallback merge byte-compares every layer of a + /// bunch — including files written outside the write channel. A user blob whose bytes + /// exactly equal the marker therefore collides with it in two ways: written through the + /// partial-update channel it is persisted as a placeholder entry, which a single-layer read + /// rejects loudly (no older layer can resolve it); left untouched in an older layer under a + /// later partial update it reads as a placeholder in every layer and silently degrades to a + /// null blob. The marker is distinctive enough that these collisions are accepted as + /// negligibly improbable. Sentinel bytes are never stored in blob files. + static constexpr char kPlaceholderSentinel[] = "_PAIMON_BLOB_PLACEHOLDER"; + /// Byte length of kPlaceholderSentinel, excluding the literal's terminating NUL. + static constexpr int32_t kPlaceholderSentinelLength = sizeof(kPlaceholderSentinel) - 1; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// reader emits kPlaceholderSentinel for placeholder entries instead of failing on them. + /// Only the data-evolution blob fallback read path sets this. + static constexpr char kEmitPlaceholderSentinelKey[] = "blob.internal.emit-placeholder-sentinel"; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// format writer persists a value exactly equal to kPlaceholderSentinel as a bin_length -2 + /// entry. Only set for data-evolution partial updates, i.e. blob-only column writes of a + /// table with data evolution enabled; all other writes store bytes verbatim. + static constexpr char kWritePlaceholderKey[] = "blob.internal.write-placeholder"; + + /// The sentinel bytes for building a data-evolution partial-update write array: a row equal + /// to this view is persisted as a placeholder entry (see kWritePlaceholderKey). + static std::string_view PlaceholderSentinelView() { + return {kPlaceholderSentinel, static_cast(kPlaceholderSentinelLength)}; + } + + /// True when the bytes are exactly the placeholder sentinel. + static bool IsPlaceholderSentinel(const char* data, size_t size) { + return size == static_cast(kPlaceholderSentinelLength) && + memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0; + } + + /// Removes the internal placeholder option keys from a format options map. The placeholder + /// channels must only ever be enabled by the internal data-evolution write and read paths, + /// so every consumer building format options from user-supplied table options strips these + /// keys before applying its own decision. + static void EraseInternalPlaceholderOptions(std::map* options) { + options->erase(kEmitPlaceholderSentinelKey); + options->erase(kWritePlaceholderKey); + } /// Blob file format version. static constexpr int8_t kFileVersion = 1; /// Magic number identifying the start of each blob bin. diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp new file mode 100644 index 000000000..0d5a6c72d --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp @@ -0,0 +1,403 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/reader/reader_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> BlobFallbackBatchReader::Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool) { + if (sequence_groups.size() < 2) { + return Status::Invalid( + "Blob fallback needs at least two sequence groups; a single group should be read " + "sequentially."); + } + if (read_schema == nullptr) { + return Status::Invalid("Blob fallback read schema cannot be nullptr."); + } + if (read_batch_size <= 0) { + return Status::Invalid(fmt::format( + "Blob fallback read batch size '{}' should be larger than zero", read_batch_size)); + } + int32_t blob_field_idx = -1; + for (int32_t i = 0; i < read_schema->num_fields(); i++) { + if (BlobUtils::IsBlobField(read_schema->field(i))) { + if (blob_field_idx != -1) { + return Status::Invalid( + "Blob fallback read schema should contain exactly one blob field."); + } + blob_field_idx = i; + } + } + if (blob_field_idx == -1) { + return Status::Invalid("Blob fallback read schema should contain a blob field."); + } + int32_t row_id_field_idx = read_schema->GetFieldIndex(SpecialFields::RowId().Name()); + int32_t seq_num_field_idx = read_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); + std::vector groups; + groups.reserve(sequence_groups.size()); + for (auto& segments : sequence_groups) { + if (segments.empty()) { + return Status::Invalid("Blob fallback sequence group should not be empty."); + } + for (const auto& segment : segments) { + if (segment.reader == nullptr && segment.gap_selected_ranges.empty()) { + return Status::Invalid( + "Blob fallback gap segment should cover at least one selected row id."); + } + } + GroupCursor cursor; + cursor.segments = std::move(segments); + groups.push_back(std::move(cursor)); + } + return std::unique_ptr( + new BlobFallbackBatchReader(std::move(groups), read_schema, blob_field_idx, + row_id_field_idx, seq_num_field_idx, read_batch_size, pool)); +} + +BlobFallbackBatchReader::BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool) + : groups_(std::move(groups)), + read_schema_(read_schema), + blob_field_idx_(blob_field_idx), + row_id_field_idx_(row_id_field_idx), + seq_num_field_idx_(seq_num_field_idx), + read_batch_size_(read_batch_size), + arrow_pool_(GetArrowPool(pool)) {} + +Result BlobFallbackBatchReader::FillWindow(size_t group_idx, int64_t want, + std::vector* chunks) { + GroupCursor& cursor = groups_[group_idx]; + int64_t collected = 0; + while (collected < want) { + if (!cursor.pending.empty()) { + const std::shared_ptr& front = cursor.pending.front(); + int64_t available = front->length() - cursor.pending_pos; + int64_t take = std::min(available, want - collected); + chunks->push_back(Chunk{front, cursor.pending_pos, take, {}}); + cursor.pending_pos += take; + collected += take; + if (cursor.pending_pos == front->length()) { + cursor.pending.pop_front(); + cursor.pending_pos = 0; + } + continue; + } + if (cursor.segment_idx >= cursor.segments.size()) { + // group exhausted; only the first group may define a shorter window + break; + } + Segment& segment = cursor.segments[cursor.segment_idx]; + if (segment.reader == nullptr) { + // gap segment: all rows are placeholders, stepped range by range so the row ids + // stay available for all-placeholder rows + if (cursor.gap_range_idx >= segment.gap_selected_ranges.size()) { + cursor.segment_idx++; + cursor.gap_range_idx = 0; + cursor.gap_range_pos = 0; + continue; + } + const Range& range = segment.gap_selected_ranges[cursor.gap_range_idx]; + int64_t remaining = range.Count() - cursor.gap_range_pos; + int64_t take = std::min(remaining, want - collected); + Chunk chunk{nullptr, 0, take, {}}; + if (row_id_field_idx_ >= 0) { + chunk.gap_row_ids.reserve(take); + for (int64_t k = 0; k < take; k++) { + chunk.gap_row_ids.push_back(range.from + cursor.gap_range_pos + k); + } + } + chunks->push_back(std::move(chunk)); + cursor.gap_range_pos += take; + collected += take; + if (cursor.gap_range_pos == range.Count()) { + cursor.gap_range_idx++; + cursor.gap_range_pos = 0; + } + continue; + } + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap, + segment.reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + cursor.segment_idx++; + continue; + } + auto& [read_batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = read_batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector selected_array_vec, + ReaderUtils::GenerateFilteredArrayVector(src_array, bitmap)); + for (const auto& selected_array : selected_array_vec) { + if (selected_array->length() == 0) { + continue; + } + auto struct_array = std::dynamic_pointer_cast(selected_array); + if (struct_array == nullptr) { + return Status::Invalid("Blob fallback expects file readers to emit struct arrays."); + } + cursor.pending.push_back(std::move(struct_array)); + } + } + return collected; +} + +Result> BlobFallbackBatchReader::ComputePlaceholderFlags( + const std::vector& chunks, int64_t row_count) const { + std::vector flags(row_count, false); + int64_t pos = 0; + for (const auto& chunk : chunks) { + if (chunk.array == nullptr) { + // gap rows stand for placeholders + std::fill(flags.begin() + pos, flags.begin() + pos + chunk.length, true); + } else { + std::shared_ptr blob_col = chunk.array->field(blob_field_idx_); + auto binary_col = std::dynamic_pointer_cast(blob_col); + if (binary_col == nullptr) { + return Status::Invalid(fmt::format( + "Blob fallback expects the blob column to be large binary, but got {}", + blob_col->type()->ToString())); + } + for (int64_t k = 0; k < chunk.length; k++) { + int64_t idx = chunk.offset + k; + if (binary_col->IsNull(idx)) { + continue; + } + std::string_view value = binary_col->GetView(idx); + if (BlobDefs::IsPlaceholderSentinel(value.data(), value.size())) { + flags[pos + k] = true; + } + } + } + pos += chunk.length; + } + return flags; +} + +Result> BlobFallbackBatchReader::AssembleRowIdRun( + const std::vector& chunks, int64_t run_start, int64_t run_end) const { + arrow::ArrayVector pieces; + int64_t pos = 0; + for (const auto& chunk : chunks) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array != nullptr) { + std::shared_ptr column = chunk.array->field(row_id_field_idx_); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } else { + arrow::Int64Builder builder(arrow_pool_.get()); + for (int64_t r = overlap_start; r < overlap_end; r++) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(chunk.gap_row_ids[r - pos])); + } + std::shared_ptr piece; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&piece)); + pieces.push_back(std::move(piece)); + } + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the + // zero-offset BatchReader contract. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result> BlobFallbackBatchReader::AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const { + const auto row_count = static_cast(group_choice.size()); + arrow::ArrayVector pieces; + int64_t run_start = 0; + while (run_start < row_count) { + const int32_t group = group_choice[run_start]; + int64_t run_end = run_start + 1; + while (run_end < row_count && group_choice[run_end] == group) { + run_end++; + } + if (group < 0) { + // placeholder in every layer: the blob degrades to null; the row keeps its row id + // (taken from the newest group, which steps in lockstep), reports -1 as its + // sequence number, and returns null for every other field + if (field_idx == row_id_field_idx_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_id_piece, + AssembleRowIdRun(group_chunks[0], run_start, run_end)); + pieces.push_back(std::move(row_id_piece)); + } else if (field_idx == seq_num_field_idx_) { + arrow::Int64Scalar seq_scalar(-1); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr seq_piece, + arrow::MakeArrayFromScalar(seq_scalar, run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(seq_piece)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_piece, + arrow::MakeArrayOfNull(read_schema_->field(field_idx)->type(), + run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(null_piece)); + } + } else { + int64_t pos = 0; + for (const auto& chunk : group_chunks[group]) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array == nullptr) { + return Status::Invalid( + "Unexpected: a gap row was chosen as a blob fallback result."); + } + std::shared_ptr column = chunk.array->field(field_idx); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + } + run_start = run_end; + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the + // zero-offset BatchReader contract. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result BlobFallbackBatchReader::NextBatch() { + if (closed_) { + return Status::Invalid("blob fallback batch reader is closed"); + } + std::vector> group_chunks(groups_.size()); + // the first (newest) group defines the window; the others must step in lockstep + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, FillWindow(0, read_batch_size_, &group_chunks[0])); + for (size_t g = 1; g < groups_.size(); g++) { + // ask for one row even when the first group is exhausted, so that a longer group is + // reported as a misalignment instead of silently truncating the read + const int64_t want = std::max(row_count, 1); + PAIMON_ASSIGN_OR_RAISE(int64_t got, FillWindow(g, want, &group_chunks[g])); + if (got != row_count) { + return Status::Invalid(fmt::format( + "All sequence groups of a blob fallback read should have the same number of " + "rows: group {} yielded {} rows in a window of {}", + g, got, row_count)); + } + } + if (row_count == 0) { + return BatchReader::MakeEofBatch(); + } + + std::vector> placeholder_flags(groups_.size()); + for (size_t g = 0; g < groups_.size(); g++) { + PAIMON_ASSIGN_OR_RAISE(placeholder_flags[g], + ComputePlaceholderFlags(group_chunks[g], row_count)); + } + // per row, the first group in max-sequence order with a real entry wins; -1 means the row is + // a placeholder in every group + std::vector group_choice(row_count, -1); + for (int64_t r = 0; r < row_count; r++) { + for (size_t g = 0; g < groups_.size(); g++) { + if (!placeholder_flags[g][r]) { + group_choice[r] = static_cast(g); + break; + } + } + } + + arrow::ArrayVector columns; + columns.reserve(read_schema_->num_fields()); + for (int32_t field_idx = 0; field_idx < read_schema_->num_fields(); field_idx++) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr column, + AssembleColumn(field_idx, group_choice, group_chunks)); + columns.push_back(std::move(column)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, + arrow::StructArray::Make(columns, read_schema_->fields())); + std::unique_ptr c_array = std::make_unique(); + std::unique_ptr c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*target_array, c_array.get(), c_schema.get())); + return std::make_pair(std::move(c_array), std::move(c_schema)); +} + +Result BlobFallbackBatchReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return BatchReader::MakeEofBatchWithBitmap(); + } + return ReaderUtils::AddAllValidBitmap(std::move(batch)); +} + +void BlobFallbackBatchReader::Close() { + for (auto& group : groups_) { + group.pending.clear(); + for (auto& segment : group.segments) { + if (segment.reader) { + segment.reader->Close(); + } + } + } + closed_ = true; +} + +std::shared_ptr BlobFallbackBatchReader::GetReaderMetrics() const { + auto metrics = std::make_shared(); + for (const auto& group : groups_) { + for (const auto& segment : group.segments) { + if (segment.reader) { + auto reader_metrics = segment.reader->GetReaderMetrics(); + if (reader_metrics) { + metrics->Merge(reader_metrics); + } + } + } + } + return metrics; +} + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.h b/src/paimon/common/reader/blob_fallback_batch_reader.h new file mode 100644 index 000000000..471054610 --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.h @@ -0,0 +1,159 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/utils/range.h" + +namespace arrow { +class Array; +class StructArray; +} // namespace arrow + +namespace paimon { + +/// Merges the blob files of one data-evolution blob bunch that span multiple max sequence +/// number layers, resolving placeholder entries row by row. Aligned with Java's +/// BlobFallbackRecordReader / AllPlaceholdersRecordReader. +/// +/// A data-evolution partial update rewrites only the touched rows of a blob column; the new blob +/// file records every untouched row as a placeholder entry. Reading therefore needs, per row, the +/// value from the newest layer that holds a real (non-placeholder) entry: +/// +/// 1. The caller groups the blob files by max sequence number (one group per layer, newest +/// first) and, inside each group, orders them by first row id. Row id ranges the group's +/// files do not cover are represented by gap segments, which stand for all-placeholder rows. +/// 2. All groups span the same overall row id range, so with the same row-ranges selection +/// applied they yield the same number of rows and can be stepped in lockstep. +/// 3. Each output row takes the first group, in max-sequence order, whose row is not a +/// placeholder. Placeholder rows are identified by exact equality with the +/// BlobDefs::kPlaceholderSentinel bytes, emitted by the blob format reader when +/// BlobDefs::kEmitPlaceholderSentinelKey is set. +/// 4. A row that is a placeholder in every group degrades to a null blob: it keeps its +/// _ROW_ID, reports -1 as its _SEQUENCE_NUMBER, and returns null for every other field. +class BlobFallbackBatchReader : public BatchReader { + public: + /// One piece of a sequence group: either a reader over a single blob file (already wrapped + /// with the usual per-file mapping readers and row selection), or a virtual gap standing for + /// row ids the group's files do not cover. Segments must be ordered by ascending row id. + struct Segment { + /// File segment: emits the rows of one blob file. Null for a gap segment. + std::unique_ptr reader; + /// Gap segment only: the selected row ids the gap emits (all placeholders), as sorted + /// disjoint ranges. Must not be empty for a gap segment. + std::vector gap_selected_ranges; + }; + + /// `sequence_groups` must be ordered by descending max sequence number and contain at least + /// two groups (a single group needs no fallback). `read_schema` is the schema every file + /// reader emits; it must contain exactly one blob field and may additionally contain the + /// row-tracking fields _ROW_ID and _SEQUENCE_NUMBER (completed per file by + /// CompleteRowTrackingFieldsBatchReader), which stay correct for rows that are a + /// placeholder in every layer. + static Result> Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool); + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + void Close() override; + + std::shared_ptr GetReaderMetrics() const override; + + private: + /// A run of consecutive rows already fetched from one group: `array` rows + /// [offset, offset + length) for a file segment, or `length` placeholder rows for a gap + /// segment (array is null). + struct Chunk { + std::shared_ptr array; + int64_t offset = 0; + int64_t length = 0; + /// Gap chunk only: the row id of each of the `length` rows, filled when the read + /// schema contains _ROW_ID so all-placeholder rows can keep their row id. + std::vector gap_row_ids; + }; + + /// Read progress of one sequence group. Move-only, matching Segment's unique_ptr member. + struct GroupCursor { + GroupCursor() = default; + GroupCursor(const GroupCursor&) = delete; + GroupCursor& operator=(const GroupCursor&) = delete; + GroupCursor(GroupCursor&&) = default; + GroupCursor& operator=(GroupCursor&&) = default; + + std::vector segments; + size_t segment_idx = 0; + /// Position inside the current gap segment: index into gap_selected_ranges and the + /// number of rows already emitted from that range. + size_t gap_range_idx = 0; + int64_t gap_range_pos = 0; + /// Rows fetched from the current file segment but not yet consumed. + std::deque> pending; + int64_t pending_pos = 0; + }; + + BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool); + + /// Collects up to `want` rows from the group into chunks. Only the first group may come up + /// short (which defines the window size); any later group ending early is a misalignment. + Result FillWindow(size_t group_idx, int64_t want, std::vector* chunks); + + /// Flags each of the `row_count` window rows of the given chunks as placeholder or not. + Result> ComputePlaceholderFlags(const std::vector& chunks, + int64_t row_count) const; + + /// Assembles one output column by stitching, per run of rows choosing the same group, + /// slices of that group's chunks. For rows choosing no group (placeholder in every layer), + /// _ROW_ID is kept, _SEQUENCE_NUMBER becomes -1, and every other field becomes null. + Result> AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const; + + /// Assembles the _ROW_ID values of rows [run_start, run_end) from the given group's chunks; + /// gap chunks contribute their synthesized row ids. + Result> AssembleRowIdRun(const std::vector& chunks, + int64_t run_start, + int64_t run_end) const; + + std::vector groups_; + std::shared_ptr read_schema_; + const int32_t blob_field_idx_; + /// Index of _ROW_ID / _SEQUENCE_NUMBER in the read schema, -1 when not read. + const int32_t row_id_field_idx_; + const int32_t seq_num_field_idx_; + const int32_t read_batch_size_; + std::shared_ptr arrow_pool_; + bool closed_ = false; +}; + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp new file mode 100644 index 000000000..c1447aa92 --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp @@ -0,0 +1,358 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/range.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +/// "PH" stands for a placeholder row (the sentinel bytes emitted by the placeholder-aware blob +/// reader), std::nullopt for null. +using BlobRows = std::vector>; + +class BlobFallbackBatchReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); + read_schema_ = arrow::schema(struct_type_->fields()); + } + + static std::string Sentinel() { + return std::string(BlobDefs::PlaceholderSentinelView()); + } + + std::shared_ptr MakeBlobStruct(const BlobRows& rows) const { + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + } + + /// One segment of a group: File(rows) for a file segment, Gap(n) for a placeholder gap + /// of n selected rows (with synthetic row ids when the schema does not read them, or the + /// given ranges via GapRanges). + struct SegmentSpec { + std::vector gap_ranges; + std::optional file_rows; + static SegmentSpec Gap(int64_t rows) { + return SegmentSpec{{Range(0, rows - 1)}, std::nullopt}; + } + static SegmentSpec GapRanges(std::vector ranges) { + return SegmentSpec{std::move(ranges), std::nullopt}; + } + static SegmentSpec File(BlobRows rows) { + return SegmentSpec{{}, std::move(rows)}; + } + }; + + std::vector MakeGroup(const std::vector& specs, + int32_t file_batch_size) const { + std::vector segments; + for (const auto& spec : specs) { + if (spec.file_rows) { + auto reader = std::make_unique(MakeBlobStruct(*spec.file_rows), + struct_type_, file_batch_size); + segments.push_back(BlobFallbackBatchReader::Segment{std::move(reader), {}}); + } else { + segments.push_back(BlobFallbackBatchReader::Segment{nullptr, spec.gap_ranges}); + } + } + return segments; + } + + /// Runs the fallback over the groups with several batch sizes and compares to expected rows. + void CheckFallback(const std::vector>& group_specs, + const BlobRows& expected_rows) const { + auto expected_array = MakeBlobStruct(expected_rows); + for (auto batch_size : arrow::internal::Iota(1, 8)) { + for (auto file_batch_size : {1, 3, 1024}) { + std::vector> groups; + groups.reserve(group_specs.size()); + for (const auto& specs : group_specs) { + groups.push_back(MakeGroup(specs, file_batch_size)); + } + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + auto expected_chunk_array = std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + + protected: + std::shared_ptr pool_; + std::shared_ptr struct_type_; + std::shared_ptr read_schema_; +}; + +TEST_F(BlobFallbackBatchReaderTest, TestBasicFallback) { + // newer layer updates row 1 only; rows 0 and 2 fall back to the older layer + CheckFallback( + {{SegmentSpec::File({"PH", "u1", "PH"})}, {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "u1", "b2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestGapPadding) { + // the newer layer only covers rows 2-3; the gaps stand for placeholders + CheckFallback({{SegmentSpec::Gap(2), SegmentSpec::File({"u2", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "b1", "u2", "b3"}); + // trailing gap + CheckFallback({{SegmentSpec::File({"PH", "u1"}), SegmentSpec::Gap(2)}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "u1", "b2", "b3"}); + // middle gap between two files of one layer + CheckFallback({{SegmentSpec::File({"u0"}), SegmentSpec::Gap(2), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"u0", "b1", "b2", "u3"}); + // a gap segment covering multiple disjoint selected ranges + CheckFallback({{SegmentSpec::GapRanges({Range(0, 0), Range(2, 2)}), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b2", "b3"})}}, + {"b0", "b2", "u3"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestAllPlaceholdersBecomesNull) { + // a row that is a placeholder in every layer degrades to null + CheckFallback({{SegmentSpec::File({"PH", "PH"})}, {SegmentSpec::File({"b0", "PH"})}}, + {"b0", std::nullopt}); + CheckFallback({{SegmentSpec::Gap(2)}, {SegmentSpec::File({"PH", "PH"})}}, + {std::nullopt, std::nullopt}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestNullIsNotPlaceholder) { + // a real null in a newer layer wins: null means "updated to null", not "not updated" + CheckFallback({{SegmentSpec::File({std::nullopt, "u1"})}, {SegmentSpec::File({"b0", "b1"})}}, + {std::nullopt, "u1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestSentinelPrefixedValueIsNotPlaceholder) { + // placeholders are identified by exact equality with the sentinel bytes only: a real value + // that merely starts with them passes through unchanged, whether it falls back or wins as + // the newest layer + std::string prefixed = Sentinel() + "suffix"; + CheckFallback({{SegmentSpec::File({"PH", "u1"})}, {SegmentSpec::File({prefixed, "b1"})}}, + {prefixed, "u1"}); + CheckFallback({{SegmentSpec::File({prefixed, "PH"})}, {SegmentSpec::File({"b0", "b1"})}}, + {prefixed, "b1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestThreeLayers) { + CheckFallback({{SegmentSpec::File({"PH", "PH", "u2"})}, + {SegmentSpec::File({"PH", "m1", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "m1", "u2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestLayeredFilesAndGaps) { + // mirrors the compacted-sequence-groups shape: layers partially cover [0, 9] + CheckFallback( + {{SegmentSpec::Gap(6), SegmentSpec::File({"u66", "PH"}), SegmentSpec::Gap(1), + SegmentSpec::File({"u69"})}, + {SegmentSpec::File({"u40", "PH", "PH", "PH"}), SegmentSpec::Gap(4), + SegmentSpec::File({"u48", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9"})}}, + {"u40", "b1", "b2", "b3", "b4", "b5", "u66", "b7", "u48", "u69"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestRowTrackingFieldsPreserved) { + // Row-tracking projections: resolved rows keep their layer's row id and sequence number; + // an all-placeholder row keeps its row id (here provided by the newest group's gap + // segment), reports -1 as its sequence number, and degrades the blob to null. Covers the + // schema variants {blob, _ROW_ID, _SEQUENCE_NUMBER}, {blob, _ROW_ID} and + // {blob, _SEQUENCE_NUMBER}. + struct RowSpec { + std::optional blob; + int64_t row_id; + int64_t seq_num; + }; + for (bool with_row_id : {true, false}) { + for (bool with_seq_num : {true, false}) { + if (!with_row_id && !with_seq_num) { + continue; + } + arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", true)}; + if (with_row_id) { + fields.push_back(SpecialFields::RowId().field_); + } + if (with_seq_num) { + fields.push_back(SpecialFields::SequenceNumber().field_); + } + auto struct_type = arrow::struct_(fields); + auto schema = arrow::schema(fields); + + auto make_rows = [&](const std::vector& rows) { + std::vector> field_builders = { + std::make_shared()}; + for (size_t i = 1; i < fields.size(); i++) { + field_builders.push_back(std::make_shared()); + } + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row.blob) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row.blob == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row.blob->data(), row.blob->size()).ok()); + } + int32_t next_field = 1; + if (with_row_id) { + auto builder = static_cast( + struct_builder.field_builder(next_field++)); + EXPECT_TRUE(builder->Append(row.row_id).ok()); + } + if (with_seq_num) { + auto builder = static_cast( + struct_builder.field_builder(next_field)); + EXPECT_TRUE(builder->Append(row.seq_num).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + }; + + for (auto batch_size : arrow::internal::Iota(1, 5)) { + for (auto file_batch_size : {1, 1024}) { + // newest layer (seq 20) covers only row 2; rows 0-1 are a gap + std::vector newest; + newest.push_back(BlobFallbackBatchReader::Segment{nullptr, {Range(0, 1)}}); + newest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique(make_rows({{"u2", 2, 20}}), + struct_type, file_batch_size), + {}}); + // oldest layer (seq 10) covers rows 0-2, row 1 is a placeholder there too + std::vector oldest; + oldest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique( + make_rows({{"b0", 0, 10}, {"PH", 1, 10}, {"PH", 2, 10}}), struct_type, + file_batch_size), + {}}); + std::vector> groups; + groups.push_back(std::move(newest)); + groups.push_back(std::move(oldest)); + + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFallbackBatchReader::Create(std::move(groups), schema, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + + // row 0 falls back to seq 10, row 1 is all-placeholder (null blob, row id + // kept, seq -1), row 2 takes seq 20 + auto expected_array = + make_rows({{"b0", 0, 10}, {std::nullopt, 1, -1}, {"u2", 2, 20}}); + auto expected_chunk_array = + std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "with_row_id=" << with_row_id << " with_seq_num=" << with_seq_num + << " batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + } +} + +TEST_F(BlobFallbackBatchReaderTest, TestMisalignedGroupsFail) { + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"PH", "u1", "PH"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b0", "b1"})}, 1024)); + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, 1024, pool_)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "same number of rows"); +} + +TEST_F(BlobFallbackBatchReaderTest, TestCreateValidation) { + // a single group needs no fallback + std::vector> single_group; + single_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(single_group), read_schema_, 1024, pool_), + "at least two sequence groups"); + + // the read schema must contain a blob field + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b1"})}, 1024)); + auto plain_schema = + arrow::schema({arrow::field("not_blob", arrow::large_binary(), /*nullable=*/true)}); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(groups), plain_schema, 1024, pool_), + "should contain a blob field"); + + // groups must not be empty + std::vector> with_empty_group; + with_empty_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_group.emplace_back(); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_group), read_schema_, 1024, pool_), + "should not be empty"); + + // a gap segment must cover at least one selected row id + std::vector> with_empty_gap; + with_empty_gap.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_gap.push_back(MakeGroup({SegmentSpec::GapRanges({})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_gap), read_schema_, 1024, pool_), + "at least one selected row id"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index a4e49ecbb..0c3a45578 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -41,8 +41,13 @@ Result> DataEvolutionFileReader::Create return Status::Invalid( "read schema, row offsets and field offsets must have the same size"); } - if (readers.size() <= 1) { - return Status::Invalid("readers size is supposed to be more than 1"); + if (readers.empty()) { + return Status::Invalid("readers must not be empty"); + } + for (int32_t reader_offset : reader_offsets) { + if (reader_offset >= static_cast(readers.size())) { + return Status::Invalid("reader offset is out of range of readers"); + } } return std::unique_ptr( new DataEvolutionFileReader(std::move(readers), read_schema, read_batch_size, diff --git a/src/paimon/common/reader/data_evolution_file_reader.h b/src/paimon/common/reader/data_evolution_file_reader.h index 2ba5853ef..36bb2641f 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.h +++ b/src/paimon/common/reader/data_evolution_file_reader.h @@ -26,10 +26,12 @@ #include "paimon/result.h" namespace paimon { -/// This is a union reader which contains multiple inner readers. +/// This is a union reader which contains one or more inner readers. /// /// This reader, assembling multiple reader into one big and great reader. The row it produces -/// also come from the readers it contains. +/// also come from the readers it contains. A single inner reader is a valid degenerate case +/// (e.g. a blob-only write): the union still maps its fields into the read schema order and +/// null fills unmatched read fields. /// /// For example, the expected schema for this reader is : int, int, string, int, string, int.(Total /// 6 fields) It contains three inner readers, we call them reader0, reader1 and reader2. diff --git a/src/paimon/common/reader/data_evolution_file_reader_test.cpp b/src/paimon/common/reader/data_evolution_file_reader_test.cpp index 4eccca7ab..3f18448c9 100644 --- a/src/paimon/common/reader/data_evolution_file_reader_test.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader_test.cpp @@ -163,6 +163,20 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) { reader_offsets, field_offsets, pool_), "read schema, row offsets and field offsets must have the same size"); } + { + arrow::FieldVector read_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::int32()), + }; + auto read_schema = arrow::schema(read_fields); + std::vector reader_offsets = {0, 0, 1, 1}; + std::vector field_offsets = {0, 1, 1, 0}; + ASSERT_NOK_WITH_MSG(DataEvolutionFileReader::Create({}, read_schema, /*read_batch_size=*/10, + reader_offsets, field_offsets, pool_), + "readers must not be empty"); + } { std::vector> readers; readers.push_back(nullptr); @@ -179,7 +193,7 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) { ASSERT_NOK_WITH_MSG( DataEvolutionFileReader::Create(std::move(readers), read_schema, /*read_batch_size=*/10, reader_offsets, field_offsets, pool_), - "readers size is supposed to be more than 1"); + "reader offset is out of range of readers"); } } diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 4b9c68e3a..3886dba9f 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -211,11 +211,11 @@ Result AppendOnlyWriter::GetDataFileWriterFacto AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const { + const std::optional>& write_cols, bool write_placeholder) const { std::shared_ptr path_factory = path_factory_; return std::make_shared(options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, - memory_pool_); + write_placeholder, memory_pool_); } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( @@ -223,8 +223,16 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri // Multiple blob fields are supported. Each blob field gets its own rolling file writer // via MultipleBlobFileWriter. auto blob_schema = schemas.blob_schema; + // A data-evolution write touching only blob columns is a partial update: its rows may mark + // untouched blobs with the placeholder sentinel (see BlobDefs::kWritePlaceholderKey). Any + // write that also carries non-blob columns stores blob bytes verbatim. The gate cannot tell + // a first write from an update, so a blob-only first write also runs under the sentinel + // channel: a user value equal to the sentinel is then persisted as a placeholder entry with + // no older layer to resolve it, and reading fails loudly rather than returning wrong bytes. + bool write_placeholder = + options_.DataEvolutionEnabled() && schemas.main_schema->num_fields() == 0; MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator = - [this, blob_schema](const std::string& blob_field_name) + [this, blob_schema, write_placeholder](const std::string& blob_field_name) -> Result< std::unique_ptr>>> { // Create a single-field schema for this blob field @@ -236,7 +244,7 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri auto single_field_schema = arrow::schema({field}); std::vector write_cols = {blob_field_name}; auto single_blob_file_writer_factory = - GetBlobFileWriterFactory(single_field_schema, write_cols); + GetBlobFileWriterFactory(single_field_schema, write_cols, write_placeholder); return std::make_unique>>( options_.GetBlobTargetFileSize(), /*target_file_row_num=*/std::numeric_limits::max(), diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 80bc18b88..a00f1e443 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -108,7 +108,7 @@ class AppendOnlyWriter : public BatchWriter { WriterFactory GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const; + const std::optional>& write_cols, bool write_placeholder) const; Status TrySyncLatestCompaction(bool blocking); Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp index aca03e7ab..c7a748270 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.cpp +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -17,8 +17,11 @@ #include "paimon/core/io/blob_data_file_writer_factory.h" #include +#include +#include #include +#include "paimon/common/data/blob_defs.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/manifest/file_source.h" @@ -33,20 +36,28 @@ BlobDataFileWriterFactory::BlobDataFileWriterFactory( const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, - const std::shared_ptr& path_factory, + const std::shared_ptr& path_factory, bool write_placeholder, const std::shared_ptr& pool) : DataFileWriterFactory(options, schema_id, pool), file_schema_(file_schema), write_cols_(write_cols), seq_num_counter_(seq_num_counter), - path_factory_(path_factory) {} + path_factory_(path_factory), + write_placeholder_(write_placeholder) {} Result>>> BlobDataFileWriterFactory::CreateWriter() const { std::shared_ptr seq_num_counter = options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; + std::map format_options = options_.ToMap(); + // The placeholder channel is internal: strip user-supplied blob.internal.* table options so + // only the writer's own decision below can enable it. + BlobDefs::EraseInternalPlaceholderOptions(&format_options); + if (write_placeholder_) { + format_options[BlobDefs::kWritePlaceholderKey] = "true"; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, - FileFormatFactory::Get("blob", options_.ToMap())); + FileFormatFactory::Get("blob", format_options)); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema_, /*create_stats_extractor=*/true)); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h index e79d42e13..3ff813f97 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.h +++ b/src/paimon/core/io/blob_data_file_writer_factory.h @@ -43,12 +43,15 @@ class BlobDataFileWriterFactory : public DataFileWriterFactory, public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { public: + /// `write_placeholder` marks a data-evolution partial-update write: the blob format + /// writer is created with BlobDefs::kWritePlaceholderKey and persists placeholder + /// sentinel rows as placeholder entries. BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + bool write_placeholder, const std::shared_ptr& pool); Result>>> CreateWriter() const override; @@ -58,6 +61,7 @@ class BlobDataFileWriterFactory std::optional> write_cols_; std::shared_ptr seq_num_counter_; std::shared_ptr path_factory_; + bool write_placeholder_ = false; }; } // namespace paimon diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp index 29b610b16..896efcf82 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp @@ -26,34 +26,46 @@ #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/object_utils.h" namespace paimon { CompleteRowTrackingFieldsBatchReader::CompleteRowTrackingFieldsBatchReader( std::unique_ptr&& reader, const std::optional& first_row_id, - int64_t snapshot_id, const std::shared_ptr& pool) + int64_t snapshot_id, const std::optional>& file_field_names, + const std::shared_ptr& pool) : first_row_id_(first_row_id), snapshot_id_(snapshot_id), + file_field_names_(file_field_names), arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)) {} Status CompleteRowTrackingFieldsBatchReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, - arrow::ImportSchema(c_file_schema.get())); + // The physical fields of the file decide which special fields must be stripped from the + // format reader's schema: a format without a self-describing file schema (e.g. blob) + // declares them via file_field_names_, self-describing formats are queried directly. + std::vector file_field_names; + if (file_field_names_) { + file_field_names = file_field_names_.value(); + } else { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, + reader_->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, + arrow::ImportSchema(c_file_schema.get())); + file_field_names = file_schema->field_names(); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(read_schema)); read_schema_ = arrow_schema; int32_t row_id_idx = arrow_schema->GetFieldIndex(SpecialFields::RowId().Name()); - if (row_id_idx != -1 && file_schema->GetFieldIndex(SpecialFields::RowId().Name()) == -1) { - // read special fields but file not exist, remove special fields to format reader + if (row_id_idx != -1 && + !ObjectUtils::Contains(file_field_names, SpecialFields::RowId().Name())) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(row_id_idx)); } int32_t sequence_id_idx = arrow_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); if (sequence_id_idx != -1 && - file_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()) == -1) { - // read special fields but file not exist, remove special fields to format reader + !ObjectUtils::Contains(file_field_names, SpecialFields::SequenceNumber().Name())) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(sequence_id_idx)); } ArrowSchema c_schema; diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.h b/src/paimon/core/io/complete_row_tracking_fields_reader.h index 2aa535d79..607e3ca5c 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.h +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.h @@ -31,10 +31,13 @@ namespace paimon { // Precondition: read_schema has special fields class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { public: - CompleteRowTrackingFieldsBatchReader(std::unique_ptr&& reader, - const std::optional& first_row_id, - int64_t snapshot_id, - const std::shared_ptr& pool); + /// `file_field_names` declares the physical fields of a file whose format has no + /// self-describing schema (e.g. blob); when nullopt the file schema is queried from the + /// inner reader via GetFileSchema(). + CompleteRowTrackingFieldsBatchReader( + std::unique_ptr&& reader, const std::optional& first_row_id, + int64_t snapshot_id, const std::optional>& file_field_names, + const std::shared_ptr& pool); Result> GetFileSchema() const override { return Status::Invalid( @@ -80,6 +83,7 @@ class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { private: std::optional first_row_id_; int64_t snapshot_id_ = -1; + std::optional> file_field_names_; std::shared_ptr arrow_pool_; std::shared_ptr read_schema_; std::unique_ptr reader_; diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp index 2ca1be8a6..1a80b08ec 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp @@ -17,7 +17,10 @@ #include "paimon/core/io/complete_row_tracking_fields_reader.h" #include +#include +#include #include +#include #include "arrow/api.h" #include "arrow/array/array_base.h" @@ -44,7 +47,8 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test { std::make_unique(src_array, src_array->type(), batch_size); auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), first_row_id, snapshot_id, pool_); + std::move(file_batch_reader), first_row_id, snapshot_id, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK(complete_row_tracking_fields_reader->SetReadSchema( @@ -63,12 +67,14 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test { void CheckSetReadSchema( const std::shared_ptr& file_schema, const std::shared_ptr& read_schema, - const std::shared_ptr& expected_schema_for_inner_reader) const { + const std::shared_ptr& expected_schema_for_inner_reader, + const std::optional>& file_field_names = std::nullopt) const { auto file_batch_reader = std::make_unique( /*data=*/nullptr, arrow::struct_(file_schema->fields()), /*read_batch_size=*/1); auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1, pool_); + std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1, + file_field_names, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( @@ -130,6 +136,33 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchema) { } } +TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchemaWithDeclaredFileFields) { + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + arrow::field("_ROW_ID", arrow::int64()), + arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + }; + { + // declared physical fields take precedence over the inner reader's self-described + // schema: the special fields are stripped even though the mock's file schema + // contains them + auto file_schema = arrow::schema(fields); + auto read_schema = arrow::schema(fields); + auto expected_schema_for_inner_reader = arrow::schema({fields[0], fields[1]}); + CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader, + std::vector{"f0", "f1"}); + } + { + // declared physical fields containing the special fields keep them for the inner reader + auto file_schema = arrow::schema({fields[0], fields[1]}); + auto read_schema = arrow::schema(fields); + auto expected_schema_for_inner_reader = read_schema; + CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader, + std::vector{"f0", "f1", "_ROW_ID", "_SEQUENCE_NUMBER"}); + } +} + TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestWithNoRowTrackingFields) { arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), @@ -342,7 +375,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidWithReadNonExistFiel auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( @@ -378,7 +412,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNextBatchBeforeSetRe auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ASSERT_NOK_WITH_MSG(complete_row_tracking_fields_reader->NextBatchWithBitmap(), "in CompleteRowTrackingFieldsBatchReader SetReadSchema is supposed to be " "called before NextBatch"); @@ -408,7 +443,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNullFirstRowId) { auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index ab323a235..4cea81edf 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -345,7 +345,8 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptr> raw_readers, split_read_->CreateRawFileReaders(partition_, {file}, read_schema_, /*predicate=*/nullptr, dv_factory_, - /*row_ranges=*/std::nullopt, data_file_path_factory_)); + /*row_ranges=*/std::nullopt, data_file_path_factory_, + /*extra_format_options=*/{})); if (raw_readers.size() != 1) { return Status::Invalid("Unexpected, CreateSstFileFromDataFile only create single reader"); } diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index cee6fd0b3..437f062f1 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -24,6 +24,7 @@ #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" @@ -75,7 +76,8 @@ Result>> AbstractSplitRead::CreateR const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const { + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const { if (data_files.empty()) { return std::vector>(); } @@ -89,7 +91,7 @@ Result>> AbstractSplitRead::CreateR auto data_file_path = data_file_path_factory->ToPath(file); PAIMON_ASSIGN_OR_RAISE(std::string data_file_identifier, file->FileFormat()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, - PrepareReaderBuilder(data_file_identifier)); + PrepareReaderBuilder(data_file_identifier, extra_format_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_reader, CreateFieldMappingReader(data_file_path, file, partition, reader_builder.get(), @@ -120,9 +122,17 @@ Result> AbstractSplitRead::ApplyPredicateFilterIfNe } Result> AbstractSplitRead::PrepareReaderBuilder( - const std::string& format_identifier) const { + const std::string& format_identifier, + const std::map& extra_format_options) const { + std::map format_options = options_.ToMap(); + // The blob placeholder channels are internal: strip user-supplied blob.internal.* table + // options so only the internal read path can enable them through extra_format_options. + BlobDefs::EraseInternalPlaceholderOptions(&format_options); + for (const auto& [key, value] : extra_format_options) { + format_options[key] = value; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format, - FileFormatFactory::Get(format_identifier, options_.ToMap())); + FileFormatFactory::Get(format_identifier, format_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, file_format->CreateReaderBuilder(options_.GetReadBatchSize())); reader_builder->WithMemoryPool(pool_); @@ -206,8 +216,15 @@ Result> AbstractSplitRead::CreateFieldMappingRe file_reader, ApplyVariantShreddingReaderIfNeeded(std::move(file_reader), read_schema)); } if (NeedCompleteRowTrackingFields(options_.RowTrackingEnabled(), read_schema)) { + // A blob file has no self-describing schema: its physical fields are declared by the + // file meta's write cols instead of queried from the format reader. + std::optional> file_field_names; + if (file_format_identifier == "blob") { + file_field_names = file_meta->write_cols; + } file_reader = std::make_unique( - std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, pool_); + std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, + file_field_names, pool_); } const auto& predicate = field_mapping->non_partition_info.non_partition_filter; auto all_data_schema = DataField::ConvertDataFieldsToArrowSchema(data_schema->Fields()); diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index 89c94fbf4..933c284d5 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -60,12 +61,16 @@ class AbstractSplitRead : public SplitRead { public: ~AbstractSplitRead() override = default; + /// `extra_format_options` are merged over the table options when building the format + /// reader, e.g. to switch the blob format reader into placeholder-aware mode for the + /// data-evolution blob fallback read path. Result>> CreateRawFileReaders( const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const; + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const; protected: AbstractSplitRead(const std::shared_ptr& path_factory, @@ -95,7 +100,8 @@ class AbstractSplitRead : public SplitRead { private: Result> PrepareReaderBuilder( - const std::string& format_identifier) const; + const std::string& format_identifier, + const std::map& extra_format_options) const; Result> CreateFileBatchReader( const std::string& file_format_identifier, const std::string& data_file_path, diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 01ed2f31f..d0c5640e8 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -16,6 +16,8 @@ #include "paimon/core/operation/data_evolution_split_read.h" +#include +#include #include #include #include @@ -28,10 +30,12 @@ #include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" #include "paimon/common/catalog/catalog_context.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h" #include "paimon/common/global_index/complete_index_score_batch_reader.h" +#include "paimon/common/reader/blob_fallback_batch_reader.h" #include "paimon/common/reader/blob_view_resolving_batch_reader.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -45,71 +49,61 @@ #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/utils/blob_view_lookup.h" namespace paimon { +int64_t DataEvolutionSplitRead::BlobBunch::RowCount() const { + if (files_.empty()) { + return 0; + } + if (!has_row_ids_selection_) { + // Add enforces the union range to be contiguous + return union_end_row_id_ - union_first_row_id_; + } + // with a row-ids selection the scan may have pruned files, leaving holes in the union + int64_t row_count = 0; + for (const auto& range : Range::SortAndMergeOverlap(ranges_, /*adjacent=*/true)) { + row_count += range.Count(); + } + return row_count; +} + Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr& file) { if (!BlobUtils::IsBlobFile(file->file_name)) { return Status::Invalid("Only blob file can be added to a blob bunch."); } PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); - if (first_row_id == latest_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with same first row id should have decreasing sequence number."); - } - // for files with the same first row id, file with larger sequence_number will be chosen, - // other files will be skipped - return Status::OK(); - } if (!files_.empty()) { - if (has_row_ids_selection_) { - // for the case: - // snapshot 1: blob0 [0, 9] - // snapshot 2: blob1 [0, 4] + blob2 [5, 9] - // when selected row id is {5}, only blob0 and blob2 is reserved in scan process, as - // blob1 has no intersect with {5} - // BlobBunch will first add blob0 [0, 9] - // then when it comes to blob2 [5, 9], blob0 will be removed as it has smaller sequence - // number - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number > latest_max_sequence_number_) { - row_count_ -= files_.back()->row_count; - files_.pop_back(); - } else { - return Status::OK(); - } - } - } else { - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with overlapping row id should have decreasing sequence " - "number."); - } - // for files with overlapping, if the file with smaller sequence_number is chosen, - // there will not exist file with larger sequence_number - return Status::OK(); - } else if (first_row_id > expected_next_first_row_id_) { - return Status::Invalid( - fmt::format("Blob file first row id should be continuous, expect {} but got {}", - expected_next_first_row_id_, first_row_id)); - } - } - if (!files_.empty()) { - // Blob files for the same field may span schema ids. - if (file->write_cols != files_[0]->write_cols) { - return Status::Invalid( - "All files in a blob bunch should have the same write columns."); - } + // Blob files for the same field may span schema ids. + if (file->write_cols != files_[0]->write_cols) { + return Status::Invalid("All files in a blob bunch should have the same write columns."); } } - row_count_ += file->row_count; - if (row_count_ > expected_row_count_) { + // files sharing a max sequence number form one layer, whose row id ranges must be disjoint; + // overlaps across layers are the expected shape of partial updates and are kept for the + // row-level placeholder fallback + auto [layer_iter, layer_inserted] = + sequence_group_end_.try_emplace(file->max_sequence_number, 0); + if (!layer_inserted && first_row_id < layer_iter->second) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping row id " + "ranges: file {} (max sequence number {}) starts at row id {} before the previous " + "file's end {}", + file->file_name, file->max_sequence_number, first_row_id, layer_iter->second)); + } + if (!has_row_ids_selection_ && !files_.empty() && first_row_id > union_end_row_id_) { + // a hole no layer covers cannot be aligned with the data files return Status::Invalid( - fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + fmt::format("Blob file first row id should be continuous, expect {} but got {}", + union_end_row_id_, first_row_id)); } + int64_t end_row_id = first_row_id + file->row_count; + layer_iter->second = end_row_id; + union_first_row_id_ = std::min(union_first_row_id_, first_row_id); + union_end_row_id_ = std::max(union_end_row_id_, end_row_id); + ranges_.emplace_back(first_row_id, end_row_id - 1); files_.push_back(file); - latest_max_sequence_number_ = file->max_sequence_number; - latest_first_row_id_ = first_row_id; - expected_next_first_row_id_ = latest_first_row_id_ + file->row_count; + if (!has_row_ids_selection_ && expected_row_count_ >= 0 && RowCount() > expected_row_count_) { + return Status::Invalid( + fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + } return Status::OK(); } DataEvolutionSplitRead::DataEvolutionSplitRead( @@ -233,7 +227,8 @@ Result> DataEvolutionSplitRead::CreateBlobViewReade std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema, /*predicate=*/nullptr, /*dv_factory=*/nullptr, - /*row_ranges=*/std::nullopt, data_file_path_factory)); + /*row_ranges=*/std::nullopt, data_file_path_factory, + /*extra_format_options=*/{})); auto batch_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -312,7 +307,8 @@ Result> DataEvolutionSplitRead::InnerCreateReader( std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_, /*predicate=*/nullptr, - /*dv_factory=*/nullptr, row_ranges, data_file_path_factory)); + /*dv_factory=*/nullptr, row_ranges, data_file_path_factory, + /*extra_format_options=*/{})); assert(raw_file_readers.size() == 1); sub_readers.push_back(std::move(raw_file_readers[0])); } else { @@ -488,18 +484,30 @@ Result> DataEvolutionSplitRead::CreateU if (!read_fields_in_file.empty()) { // create new FieldMappingReader for read partial fields auto file_read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields_in_file); - PAIMON_ASSIGN_OR_RAISE(std::vector> file_readers, - CreateRawFileReaders(partition, bunch->Files(), file_read_schema, - /*predicate=*/nullptr, /*dv_factory=*/{}, - row_ranges, data_file_path_factory)); - if (file_readers.size() == 1) { - file_batch_readers[file_idx] = std::move(file_readers[0]); + auto blob_bunch = std::dynamic_pointer_cast(bunch); + if (blob_bunch && !blob_bunch->SequentialReadOptimize()) { + // blob files span multiple max sequence number layers: placeholder entries of + // newer layers must fall back row by row to older layers + PAIMON_ASSIGN_OR_RAISE( + file_batch_readers[file_idx], + CreateBlobFallbackReader(partition, blob_bunch->Files(), file_read_schema, + row_ranges, data_file_path_factory)); } else { - auto raw_readers = - ObjectUtils::MoveVector>(std::move(file_readers)); - // Concat multiple blob files that map to the same data file. - file_batch_readers[file_idx] = - std::make_unique(std::move(raw_readers), pool_); + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, bunch->Files(), file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, + /*extra_format_options=*/{})); + if (file_readers.size() == 1) { + file_batch_readers[file_idx] = std::move(file_readers[0]); + } else { + auto raw_readers = ObjectUtils::MoveVector>( + std::move(file_readers)); + // Concat multiple blob files that map to the same data file. + file_batch_readers[file_idx] = + std::make_unique(std::move(raw_readers), pool_); + } } } } @@ -509,6 +517,98 @@ Result> DataEvolutionSplitRead::CreateU field_offsets, pool_); } +namespace { +/// Selected row ids in [from, to] as sorted disjoint ranges: the whole range without a +/// selection, otherwise its intersection with the (possibly overlapping) selected ranges. +std::vector SelectedRangesInRange(int64_t from, int64_t to, + const std::optional>& row_ranges) { + if (!row_ranges) { + return {Range(from, to)}; + } + std::vector selected; + Range gap_range(from, to); + for (const auto& range : Range::SortAndMergeOverlap(row_ranges.value(), /*adjacent=*/true)) { + std::optional intersection = Range::Intersection(gap_range, range); + if (intersection) { + selected.push_back(*intersection); + } + } + return selected; +} +} // namespace + +Result> DataEvolutionSplitRead::CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const { + int64_t union_first_row_id = std::numeric_limits::max(); + int64_t union_last_row_id = std::numeric_limits::min(); + using FileWithFirstRowId = std::pair>; + std::map, std::greater<>> sequence_groups; + for (const auto& file : files) { + PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); + union_first_row_id = std::min(union_first_row_id, first_row_id); + union_last_row_id = std::max(union_last_row_id, first_row_id + file->row_count - 1); + sequence_groups[file->max_sequence_number].emplace_back(first_row_id, file); + } + // the blob format reader must emit placeholder sentinels instead of failing on them + const std::map blob_format_options = { + {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + std::vector> groups; + groups.reserve(sequence_groups.size()); + for (auto& [max_sequence_number, group_files] : sequence_groups) { + std::stable_sort(group_files.begin(), group_files.end(), + [](const FileWithFirstRowId& f1, const FileWithFirstRowId& f2) { + return f1.first < f2.first; + }); + std::vector segments; + // pad row ids this layer does not cover with placeholder gaps, so that every layer spans + // the same union range and the groups can be stepped in lockstep + int64_t next_row_id = union_first_row_id; + for (const auto& [first_row_id, file] : group_files) { + if (first_row_id < next_row_id) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping " + "row id ranges: file {} (max sequence number {}) starts at row id {} before " + "the previous file's end {}", + file->file_name, max_sequence_number, first_row_id, next_row_id)); + } + if (first_row_id > next_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, first_row_id - 1, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, {file}, file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, blob_format_options)); + if (file_readers.size() != 1) { + return Status::Invalid("Unexpected: blob fallback file reader was skipped."); + } + segments.push_back(BlobFallbackBatchReader::Segment{std::move(file_readers[0]), {}}); + next_row_id = first_row_id + file->row_count; + } + if (next_row_id <= union_last_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, union_last_row_id, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + groups.push_back(std::move(segments)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_reader, + BlobFallbackBatchReader::Create(std::move(groups), file_read_schema, + options_.GetReadBatchSize(), pool_)); + return std::move(fallback_reader); +} + Result DataEvolutionSplitRead::Match(const std::shared_ptr& split, bool force_keep_delete) const { return true; diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index c3ed5033f..87b602602 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -55,7 +57,8 @@ struct DeletionFile; /// Readers Overview: (ConcatBatchReader across /// splits)->(BlobViewResolvingBatchReader)->(CompleteIndexScoreBatchReader)-> /// CompleteRowKindBatchReader->(PredicateBatchReader) -/// ->ConcatBatchReader across files->DataEvolutionFileReader->(ConcatBatchReader across blob files) +/// ->ConcatBatchReader across files->DataEvolutionFileReader +/// ->(ConcatBatchReader across blob files | BlobFallbackBatchReader across blob sequence layers) /// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader) /// ->(MapSharedShreddingFileReader) /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader @@ -107,26 +110,43 @@ class DataEvolutionSplitRead : public AbstractSplitRead { std::vector> data_files_; }; + /// All blob files of one blob field in a merge group. Unlike data files, every file is kept + /// (aligned with Java's BlobFileBunch): files whose max sequence numbers differ form layers + /// of a data-evolution partial update, where newer layers record untouched rows as + /// placeholder entries and reading falls back row by row to older layers. + /// Files must be added ordered by first row id ascending (then max sequence number + /// descending), as produced by MergeRangesAndSort. class BlobBunch : public FieldBunch { public: explicit BlobBunch(int64_t expected_row_count, bool has_row_ids_selection) : expected_row_count_(expected_row_count), has_row_ids_selection_(has_row_ids_selection) {} - int64_t RowCount() const override { - return row_count_; - } + /// Number of distinct row ids covered by the added files. Without a row-ids selection + /// the covered range is contiguous (enforced by Add) and matches the data files. + int64_t RowCount() const override; const std::vector>& Files() const override { return files_; } Status Add(const std::shared_ptr& file); + /// True when every file shares one max sequence number, so the files are read + /// sequentially without the fallback merge. A lone layer is expected to hold no + /// placeholder entries; if one does (a user value equal to the placeholder sentinel + /// written by a blob-only first write), the strict blob reader rejects the read, since + /// no older layer exists to resolve it. + bool SequentialReadOptimize() const { + return sequence_group_end_.size() <= 1; + } private: int64_t expected_row_count_ = -1; - int64_t latest_first_row_id_ = -1; - int64_t expected_next_first_row_id_ = -1; - int64_t latest_max_sequence_number_ = -1; - int64_t row_count_ = 0; bool has_row_ids_selection_ = false; + int64_t union_first_row_id_ = std::numeric_limits::max(); + /// Exclusive end of the union row id range covered so far. + int64_t union_end_row_id_ = std::numeric_limits::min(); + /// Per max sequence number: exclusive end of the last added range, to reject + /// overlapping files within one layer. + std::map sequence_group_end_; + std::vector ranges_; std::vector> files_; }; @@ -164,6 +184,16 @@ class DataEvolutionSplitRead : public AbstractSplitRead { const std::vector>& need_merge_files, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; + + /// Builds the row-level fallback reader for a blob bunch spanning multiple max sequence + /// number layers: groups the files by max sequence number, pads uncovered row id ranges of + /// each layer with placeholder gap segments, and resolves each row to the newest + /// non-placeholder layer. See BlobFallbackBatchReader. + Result> CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const; }; } // namespace paimon diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index aa6f9c9e1..4cf3a664b 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -145,72 +145,88 @@ TEST_F(DataEvolutionSplitReadTest, TestAddNonBlobFileInvalid) { TEST_F(DataEvolutionSplitReadTest, TestAddBlobWithSameFirstRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/1, + /*max_sequence_number=*/3, /*write_cols=*/std::optional>({"blob_col"})); - auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + auto blob_full_tail = + CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); + auto blob_short_tail = + CreateBlobFile("blob3", /*first_row_id=*/0, /*row_count=*/50, + /*max_sequence_number=*/1, + /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), - "Blob file with same first row id should have decreasing sequence number."); + // Files with the same first row id and lower sequence numbers are older layers of a + // partial update; they are kept for the row-level placeholder fallback, whether they + // cover the same range or only a shorter prefix of it. + ASSERT_OK(blob_bunch->Add(blob_full_tail)); + ASSERT_OK(blob_bunch->Add(blob_short_tail)); + + ASSERT_EQ(blob_bunch->Files().size(), 3); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_full_tail); + ASSERT_EQ(blob_bunch->Files()[2], blob_short_tail); + ASSERT_EQ(blob_bunch->RowCount(), 100); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithSameFirstRowIdAndLowerSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with same firstRowId and lower sequence number should be ignored + // Overlapping layers with different sequence numbers are kept for the fallback. ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/1, + /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with overlapping row id and lower sequence number should be ignored ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdInSameLayer) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG( - blob_bunch->Add(blob_tail), - "Blob file with overlapping row id should have decreasing sequence number."); + // Files sharing a max sequence number form one layer and must not overlap. + ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), + "Blob files with the same max sequence number should not have overlapping " + "row id ranges"); } TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithNonContinuousRowId) { @@ -265,12 +281,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, blob_entry is removed + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is a newer + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_sub2); - ASSERT_EQ(blob_bunch->RowCount(), 5); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); + ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { @@ -291,13 +309,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, as blob_sub2 has smaller sequence number, blob_sub2 will be - // skipped. + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is an older + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); + ASSERT_EQ(blob_bunch->Files().size(), 2); ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelection) { @@ -420,15 +439,15 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { std::vector> batch = batches[0]; ASSERT_EQ(batch.size(), 10); ASSERT_EQ(batch[0], data); - ASSERT_EQ(batch[1], blob_entry5); // pick - ASSERT_EQ(batch[2], blob_entry2); // skip - ASSERT_EQ(batch[3], blob_entry1); // skip - ASSERT_EQ(batch[4], blob_entry9); // pick - ASSERT_EQ(batch[5], blob_entry6); // skip - ASSERT_EQ(batch[6], blob_entry3); // skip - ASSERT_EQ(batch[7], blob_entry7); // pick - ASSERT_EQ(batch[8], blob_entry4); // skip - ASSERT_EQ(batch[9], blob_entry8); // pick + ASSERT_EQ(batch[1], blob_entry5); + ASSERT_EQ(batch[2], blob_entry2); + ASSERT_EQ(batch[3], blob_entry1); + ASSERT_EQ(batch[4], blob_entry9); + ASSERT_EQ(batch[5], blob_entry6); + ASSERT_EQ(batch[6], blob_entry3); + ASSERT_EQ(batch[7], blob_entry7); + ASSERT_EQ(batch[8], blob_entry4); + ASSERT_EQ(batch[9], blob_entry8); auto blob_field_to_field_id = [](const std::shared_ptr&) -> Result { return 0; @@ -440,12 +459,19 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { ASSERT_EQ(bunch.size(), 2); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { @@ -561,20 +587,33 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { ASSERT_EQ(bunch.size(), 3); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); auto blob_bunch2 = std::dynamic_pointer_cast(bunch[2]); - ASSERT_EQ(blob_bunch2->Files().size(), 4); + ASSERT_EQ(blob_bunch2->Files().size(), 9); ASSERT_EQ(blob_bunch2->Files()[0], blob_entry15); - ASSERT_EQ(blob_bunch2->Files()[1], blob_entry19); - ASSERT_EQ(blob_bunch2->Files()[2], blob_entry17); - ASSERT_EQ(blob_bunch2->Files()[3], blob_entry18); + ASSERT_EQ(blob_bunch2->Files()[1], blob_entry12); + ASSERT_EQ(blob_bunch2->Files()[2], blob_entry11); + ASSERT_EQ(blob_bunch2->Files()[3], blob_entry19); + ASSERT_EQ(blob_bunch2->Files()[4], blob_entry16); + ASSERT_EQ(blob_bunch2->Files()[5], blob_entry13); + ASSERT_EQ(blob_bunch2->Files()[6], blob_entry17); + ASSERT_EQ(blob_bunch2->Files()[7], blob_entry14); + ASSERT_EQ(blob_bunch2->Files()[8], blob_entry18); ASSERT_EQ(blob_bunch2->RowCount(), 1000); + ASSERT_FALSE(blob_bunch2->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestDifferentRowIdRange) { diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index 86df4841d..e1cee5a08 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -272,7 +272,8 @@ Result> MergeFileSplitRead::CreateNoMergeReader( std::vector> raw_file_readers, CreateRawFileReaders(data_split->Partition(), data_split->DataFiles(), read_schema, only_filter_key ? predicate_for_keys_ : context_->GetPredicate(), - dv_factory, /*row_ranges=*/{}, data_file_path_factory)); + dv_factory, /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -496,7 +497,8 @@ Result> MergeFileSplitRead::CreateReaderFo PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); assert(data_files.size() == raw_file_readers.size()); // KeyValueDataFileRecordReader converts arrow array from format reader to KeyValue objects diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index ff213a4ea..c027e1c72 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -81,7 +81,8 @@ Result> RawFileSplitRead::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.cpp b/src/paimon/core/table/source/data_evolution_batch_scan.cpp index 94310b2e5..cabb03d85 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan.cpp +++ b/src/paimon/core/table/source/data_evolution_batch_scan.cpp @@ -16,6 +16,9 @@ #include "paimon/core/table/source/data_evolution_batch_scan.h" +#include +#include + #include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/table/source/data_split_impl.h" @@ -72,7 +75,7 @@ Result> DataEvolutionBatchScan::CreatePlan() { Result> DataEvolutionBatchScan::WrapToIndexedSplits( const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index, - const std::map& id_to_score) const { + const std::map& id_to_score) { // TODO(lisizhuo.lsz): add executor here auto data_splits = data_plan->Splits(); std::vector> indexed_splits; @@ -86,11 +89,23 @@ Result> DataEvolutionBatchScan::WrapToIndexedSplits( if (files.empty()) { return Status::Invalid("Empty data files in WrapToIndexedSplits"); } - PAIMON_ASSIGN_OR_RAISE(int64_t min, files[0]->NonNullFirstRowId()); - PAIMON_ASSIGN_OR_RAISE(int64_t max, files[files.size() - 1]->NonNullFirstRowId()); - max += files[files.size() - 1]->row_count - 1; + // The row-id ranges of the files in a split may be unordered, discontiguous, or + // overlapping, so intersect the index with each file's range separately, then sort and + // merge the intersected ranges. + std::vector intersected; + int64_t min = std::numeric_limits::max(); + int64_t max = std::numeric_limits::min(); + for (const auto& file : files) { + PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); + int64_t last_row_id = first_row_id + file->row_count - 1; + min = std::min(min, first_row_id); + max = std::max(max, last_row_id); + std::vector file_ranges = + row_range_index.IntersectedRanges(first_row_id, last_row_id); + intersected.insert(intersected.end(), file_ranges.begin(), file_ranges.end()); + } - std::vector expected = row_range_index.IntersectedRanges(min, max); + std::vector expected = Range::SortAndMergeOverlap(intersected, /*adjacent=*/true); if (expected.empty()) { return Status::Invalid( fmt::format("There should be intersected ranges for split with min row id {} and " diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.h b/src/paimon/core/table/source/data_evolution_batch_scan.h index 88f20f762..306a25159 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan.h +++ b/src/paimon/core/table/source/data_evolution_batch_scan.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -37,10 +38,14 @@ class DataEvolutionBatchScan : public AbstractTableScan { Result> CreatePlan() override; - private: - Result> WrapToIndexedSplits( + /// Wraps each DataSplit in `data_plan` into an IndexedSplit whose row ranges are the + /// intersection of `row_range_index` and the row-id range of each data file. Visible for + /// testing. + static Result> WrapToIndexedSplits( const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index, - const std::map& id_to_score) const; + const std::map& id_to_score); + + private: Result> EvalGlobalIndex() const; private: diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp new file mode 100644 index 000000000..9081d7fd0 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp @@ -0,0 +1,113 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/source/data_evolution_batch_scan.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/plan_impl.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon::test { +namespace { +std::shared_ptr NewAppendFile(const std::string& file_name, int64_t first_row_id, + int64_t row_count) { + return std::make_shared( + file_name, /*file_size=*/1024l, row_count, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_sequence_number=*/0l, + /*max_sequence_number=*/first_row_id + row_count - 1, /*schema_id=*/0, /*level=*/0, + std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0, + /*embedded_index=*/nullptr, FileSource::Append(), std::nullopt, std::nullopt, first_row_id, + std::nullopt); +} + +std::shared_ptr NewDataPlan(std::vector> files) { + DataSplitImpl::Builder builder( + /*partition=*/BinaryRow::EmptyRow(), + /*bucket=*/0, /*bucket_path=*/"data/test_table/bucket-0", std::move(files)); + std::shared_ptr data_split = + builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value(); + return std::make_shared(/*snapshot_id=*/1, + std::vector>({data_split})); +} +} // namespace + +TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithUnorderedAndDiscontiguousDataFiles) { + // The files cover [4650, 4700], [4300, 4450] and [4200, 4407]: unordered, partially + // overlapping, with a gap [4451, 4649] that must not appear in the wrapped ranges. + std::shared_ptr data_plan = + NewDataPlan({NewAppendFile("file-1", 4650l, 51l), NewAppendFile("file-2", 4300l, 151l), + NewAppendFile("file-3", 4200l, 208l)}); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 5000)})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, + DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index, + /*id_to_score=*/{})); + + ASSERT_EQ(plan->Splits().size(), 1); + auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_NE(indexed_split, nullptr); + ASSERT_EQ(indexed_split->GetDataSplit(), data_plan->Splits()[0]); + ASSERT_EQ(indexed_split->RowRanges(), + std::vector({Range(4200, 4450), Range(4650, 4700)})); + ASSERT_TRUE(indexed_split->Scores().empty()); +} + +TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsExcludesRowIdsInFileRangeGaps) { + // The split covers [0, 9] and [20, 29] while the index hits {5, 15, 25}. Row id 15 lies in + // the gap between the two files and must be excluded, together with its score. + std::shared_ptr data_plan = + NewDataPlan({NewAppendFile("file-1", 0l, 10l), NewAppendFile("file-2", 20l, 10l)}); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, + RowRangeIndex::Create({Range(5, 5), Range(15, 15), Range(25, 25)})); + std::map id_to_score = {{5l, 0.5f}, {15l, 0.7f}, {25l, 0.9f}}; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, DataEvolutionBatchScan::WrapToIndexedSplits( + data_plan, row_range_index, id_to_score)); + + ASSERT_EQ(plan->Splits().size(), 1); + auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_NE(indexed_split, nullptr); + ASSERT_EQ(indexed_split->RowRanges(), std::vector({Range(5, 5), Range(25, 25)})); + ASSERT_EQ(indexed_split->Scores(), std::vector({0.5f, 0.9f})); +} + +TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithoutIntersection) { + std::shared_ptr data_plan = NewDataPlan({NewAppendFile("file-1", 100l, 10l)}); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 50)})); + + ASSERT_NOK_WITH_MSG(DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index, + /*id_to_score=*/{}), + "There should be intersected ranges for split with min row id 100 and " + "max row id 109."); +} + +} // namespace paimon::test diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index d5b5a2fe2..43156fe3e 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -17,6 +17,7 @@ #include "paimon/format/blob/blob_file_batch_reader.h" #include +#include #include #include "arrow/api.h" @@ -38,7 +39,7 @@ namespace paimon::blob { Result> BlobFileBatchReader::Create( const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor, - const std::shared_ptr& pool) { + bool emit_placeholder_sentinel, const std::shared_ptr& pool) { if (input_stream == nullptr) { return Status::Invalid("blob file batch reader create failed: input stream is nullptr"); } @@ -83,14 +84,15 @@ Result> BlobFileBatchReader::Create( int64_t offset = 0; for (const auto& blob_length : blob_lengths) { blob_offsets.push_back(offset); - // Null blobs (bin_length == -1) don't occupy file space + // null (-1) and placeholder (-2) entries occupy no file space if (blob_length >= 0) { offset += blob_length; } } PAIMON_ASSIGN_OR_RAISE(std::string file_path, input_stream->GetUri()); - auto reader = std::unique_ptr(new BlobFileBatchReader( - input_stream, file_path, blob_lengths, blob_offsets, batch_size, blob_as_descriptor, pool)); + auto reader = std::unique_ptr( + new BlobFileBatchReader(input_stream, file_path, blob_lengths, blob_offsets, batch_size, + blob_as_descriptor, emit_placeholder_sentinel, pool)); return reader; } @@ -99,6 +101,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, bool blob_as_descriptor, + bool emit_placeholder_sentinel, const std::shared_ptr& pool) : input_stream_(input_stream), file_path_(file_path), @@ -108,6 +111,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp target_blob_offsets_(blob_offsets), batch_size_(batch_size), blob_as_descriptor_(blob_as_descriptor), + emit_placeholder_sentinel_(emit_placeholder_sentinel), pool_(pool), arrow_pool_(GetArrowPool(pool_)), metrics_(std::make_shared()) { @@ -156,7 +160,7 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, } target_type_ = arrow::struct_(arrow_schema->fields()); current_pos_ = 0; - previous_batch_first_row_number_ = std::numeric_limits::max(); + previous_batch_start_pos_ = std::numeric_limits::max(); previous_batch_row_count_ = 0; return Status::OK(); } @@ -168,11 +172,7 @@ Result> BlobFileBatchReader::NextBlobOffsets( PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(0)); int64_t data_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - // Null blobs contribute zero bytes to content - if (!IsTargetNull(i)) { - data_length += GetTargetContentLength(i); - } + data_length += GetTargetOutputLength(current_pos_ + k); PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(data_length)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr offset_buffer, @@ -184,10 +184,7 @@ Result> BlobFileBatchReader::NextBlobContents( int32_t rows_to_read) const { int64_t total_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - if (!IsTargetNull(i)) { - total_length += GetTargetContentLength(i); - } + total_length += GetTargetOutputLength(current_pos_ + k); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr data_buffer, arrow::AllocateBuffer(total_length, arrow_pool_.get())); @@ -197,6 +194,13 @@ Result> BlobFileBatchReader::NextBlobContents( if (IsTargetNull(i)) { continue; } + if (IsTargetPlaceholder(i)) { + // a placeholder entry has no data bytes in the file; emit the sentinel for the + // data-evolution blob fallback merge to identify it + memcpy(buffer, BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength); + buffer += BlobDefs::kPlaceholderSentinelLength; + continue; + } int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); PAIMON_RETURN_NOT_OK(ReadBlobContentAt(offset, length, buffer)); @@ -268,6 +272,9 @@ Result> BlobFileBatchReader::BuildTargetArray( PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append()); if (IsTargetNull(i)) { PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull()); + } else if (IsTargetPlaceholder(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append( + BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength)); } else { int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); @@ -291,18 +298,29 @@ Result BlobFileBatchReader::NextBatch() { return Status::Invalid("target type is nullptr, call SetReadSchema first"); } if (current_pos_ >= target_blob_lengths_.size()) { - PAIMON_ASSIGN_OR_RAISE(previous_batch_first_row_number_, GetNumberOfRows()); + previous_batch_start_pos_ = target_blob_lengths_.size(); previous_batch_row_count_ = 0; return BatchReader::MakeEofBatch(); } int32_t left_rows = target_blob_lengths_.size() - current_pos_; int32_t rows_to_read = std::min(left_rows, batch_size_); + if (!emit_placeholder_sentinel_) { + for (int32_t k = 0; k < rows_to_read; ++k) { + if (IsTargetPlaceholder(current_pos_ + k)) { + return Status::Invalid(fmt::format( + "blob file {} contains a placeholder entry (bin_length {}) written by a " + "data-evolution partial update; it can only be resolved by the data-evolution " + "blob fallback read path", + file_path_, BlobDefs::kPlaceholderBinLength)); + } + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob_array, BuildTargetArray(rows_to_read)); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*blob_array, c_array.get(), c_schema.get())); - previous_batch_first_row_number_ = target_blob_row_indexes_[current_pos_]; + previous_batch_start_pos_ = current_pos_; current_pos_ += rows_to_read; previous_batch_row_count_ = c_array->length; return make_pair(std::move(c_array), std::move(c_schema)); diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 27549c190..7747b60c8 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -49,7 +49,8 @@ namespace paimon::blob { /// ==================================================================== /// 1. Data Bins Section /// ==================================================================== -/// The file consists of one or more contiguous 'bins' (bin_0, bin_1, bin_2, ...). +/// The file consists of zero or more contiguous 'bins' (bin_0, bin_1, bin_2, ...); rows whose +/// index entry is negative (see Section 2) have no bin. /// The structure of each bin is as follows: /// /// | Field Name | Length (bytes) | Description | @@ -67,7 +68,13 @@ namespace paimon::blob { /// ==================================================================== /// The Index is located after all data bins and is used for quick lookup and management. /// -/// Purpose: Records the lengths (record lens) of all data bins. +/// Purpose: Records one signed length entry per row, in row order. +/// +/// Special lengths: an entry does not always describe a bin present in the Data Bins Section. +/// - -1 (BlobDefs::kNullBinLength): a null blob; no bin is written. +/// - -2 (BlobDefs::kPlaceholderBinLength): a placeholder blob written by a data-evolution +/// partial update for a row it did not touch; no bin is written and the value must be +/// resolved from an older blob file covering the same row. /// /// Encoding: /// - Uses Delta Encoding to store differences between successive length values. @@ -87,9 +94,16 @@ namespace paimon::blob { /// - Current version is 1. class BlobFileBatchReader : public FileBatchReader { public: + /// `emit_placeholder_sentinel` controls how placeholder entries (bin_length == + /// BlobDefs::kPlaceholderBinLength) are read: when false they fail the read, as resolving + /// them requires the data-evolution blob fallback path; when true they are returned as the + /// non-null BlobDefs::kPlaceholderSentinel bytes for that path to merge away. Stored values + /// are returned verbatim; see BlobDefs::kPlaceholderSentinel for the accepted collision + /// with a user value exactly equal to the sentinel. static Result> Create( const std::shared_ptr& input_stream, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Result> GetFileSchema() const override; @@ -99,14 +113,8 @@ class BlobFileBatchReader : public FileBatchReader { Result NextBatch() override; Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { - if (all_blob_lengths_.size() != target_blob_lengths_.size()) { - return Status::NotImplemented( - "Cannot call GetPreviousBatchFileRowId in BlobFileBatchReader because, after " - "bitmap pushdown, rows in the array returned by NextBatch are no longer " - "contiguous."); - } if (previous_batch_row_count_ == 0) { - if (previous_batch_first_row_number_ == std::numeric_limits::max()) { + if (previous_batch_start_pos_ == std::numeric_limits::max()) { return Status::Invalid("No batch has been read yet."); } else { return Status::Invalid("Last batch was EOF."); @@ -117,7 +125,9 @@ class BlobFileBatchReader : public FileBatchReader { fmt::format("batch_row_id {} is out of range, last batch row count is {}", batch_row_id, previous_batch_row_count_)); } - return previous_batch_first_row_number_ + batch_row_id; + // target_blob_row_indexes_ maps every selected position back to its original file row + // index, so this stays correct after a bitmap selection removed rows. + return target_blob_row_indexes_[previous_batch_start_pos_ + batch_row_id]; } Result GetNumberOfRows() const override { @@ -144,7 +154,8 @@ class BlobFileBatchReader : public FileBatchReader { BlobFileBatchReader(const std::shared_ptr& input_stream, const std::string& file_path, const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Status ReadBlobContentAt(const int64_t offset, const int64_t length, uint8_t* content) const; @@ -160,6 +171,23 @@ class BlobFileBatchReader : public FileBatchReader { return target_blob_lengths_[index] == BlobDefs::kNullBinLength; } + bool IsTargetPlaceholder(size_t index) const { + return target_blob_lengths_[index] == BlobDefs::kPlaceholderBinLength; + } + + /// Content bytes the blob at the given index contributes to the output buffer: nothing for + /// a null entry, the sentinel bytes for a placeholder entry (which occupies no file space), + /// the stored bytes otherwise. + int64_t GetTargetOutputLength(size_t index) const { + if (IsTargetNull(index)) { + return 0; + } + if (IsTargetPlaceholder(index)) { + return BlobDefs::kPlaceholderSentinelLength; + } + return GetTargetContentLength(index); + } + int64_t GetTargetContentOffset(size_t index) const { return target_blob_offsets_[index] + BlobDefs::kContentStartOffset; } @@ -179,6 +207,7 @@ class BlobFileBatchReader : public FileBatchReader { const int32_t batch_size_; const bool blob_as_descriptor_; + const bool emit_placeholder_sentinel_; std::shared_ptr pool_; std::shared_ptr arrow_pool_; @@ -186,7 +215,9 @@ class BlobFileBatchReader : public FileBatchReader { std::shared_ptr metrics_; size_t current_pos_ = 0; - uint64_t previous_batch_first_row_number_ = std::numeric_limits::max(); + /// Start position of the previous batch in the (possibly selection-filtered) target index + /// space; max() until the first batch is read. + size_t previous_batch_start_pos_ = std::numeric_limits::max(); uint64_t previous_batch_row_count_ = 0; bool closed_ = false; }; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 4e53dfb4c..866e136c2 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -44,9 +44,9 @@ class BlobFileBatchReaderTest : public testing::Test, public ::testing::WithPara std::shared_ptr fs = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/" + paimon_blob_file)); - ASSERT_OK_AND_ASSIGN(auto reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, - blob_as_descriptor, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( + input_stream, /*batch_size=*/1024, blob_as_descriptor, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection_bitmap)); ASSERT_OK_AND_ASSIGN(auto chunked_array, paimon::test::ReadResultCollector::CollectResult(reader.get())); @@ -162,9 +162,10 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_OK_AND_ASSIGN( std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -187,6 +188,45 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); } +TEST_F(BlobFileBatchReaderTest, TestRowNumbersWithSelectionBitmap) { + // a selection bitmap removes rows, but batch positions must still map back to the original + // file row indexes so _ROW_ID completion works under row-range pushdown + auto schema = arrow::schema({BlobUtils::ToArrowField("my_blob_field", false)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + + std::string test_data_path = paimon::test::GetDataDir() + "/db_with_blob.db/table_with_blob/"; + auto dir = paimon::test::UniqueTestDirectory::Create(); + std::string table_path = dir->Str(); + ASSERT_TRUE(paimon::test::TestUtil::CopyDirectory(test_data_path, table_path)); + + std::shared_ptr fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr input_stream, + fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); + + RoaringBitmap32 selection; + selection.Add(0); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection)); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "No batch has been read yet"); + ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch()); + ArrowArrayRelease(batch1.first.get()); + ArrowSchemaRelease(batch1.second.get()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); + ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch()); + ArrowArrayRelease(batch2.first.get()); + ArrowSchemaRelease(batch2.second.get()); + ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value()); + ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch()); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "Last batch was EOF"); + ASSERT_TRUE(BatchReader::IsEofBatch(batch3)); +} + TEST_F(BlobFileBatchReaderTest, InvalidScenario) { auto dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -202,20 +242,22 @@ TEST_F(BlobFileBatchReaderTest, InvalidScenario) { { ASSERT_NOK_WITH_MSG( BlobFileBatchReader::Create(input_stream, - /*batch_size=*/0, /*blob_as_descriptor=*/true, pool_), + /*batch_size=*/0, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_), "blob file batch reader create failed: read batch size '0' should be larger than zero"); } { ASSERT_NOK_WITH_MSG( BlobFileBatchReader::Create(/*input_stream=*/nullptr, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_), + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_), "blob file batch reader create failed: input stream is nullptr"); } { ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(/*input_stream=*/input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(/*input_stream=*/input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->GetFileSchema(), "blob file has no self-describing file schema"); ASSERT_TRUE(reader->GetReaderMetrics()); @@ -237,7 +279,8 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream, struct_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -248,9 +291,10 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(dir->Str() + "/file.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -273,9 +317,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(/*read_schema=*/nullptr, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "SetReadSchema failed: read schema cannot be nullptr"); @@ -298,9 +342,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "read schema field number 2 is not 1"); @@ -323,9 +367,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "field my_blob_field: large_binary is not BLOB"); @@ -346,9 +390,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); RoaringBitmap32 roaring; roaring.Add(0); roaring.Add(1); diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index 875566ffc..b98e56c55 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -39,7 +39,7 @@ namespace paimon::blob { BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, - bool write_null_on_fetch_failure, + bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) : out_(out), @@ -48,7 +48,8 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con fs_(fs), pool_(pool), write_null_on_missing_file_(write_null_on_missing_file), - write_null_on_fetch_failure_(write_null_on_fetch_failure) { + write_null_on_fetch_failure_(write_null_on_fetch_failure), + write_placeholder_(write_placeholder) { // Create() has already checked that data_type has exactly one BLOB field. blob_field_name_ = data_type_->field(0)->name(); metrics_ = std::make_shared(); @@ -59,7 +60,7 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con Result> BlobFormatWriter::Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); @@ -82,8 +83,9 @@ Result> BlobFormatWriter::Create( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); - return std::unique_ptr(new BlobFormatWriter( - out, uri, data_type, write_null_on_missing_file, write_null_on_fetch_failure, fs, pool)); + return std::unique_ptr( + new BlobFormatWriter(out, uri, data_type, write_null_on_missing_file, + write_null_on_fetch_failure, write_placeholder, fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -117,7 +119,16 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { const auto& blob_array = arrow::internal::checked_cast(*child_array); assert(blob_array.length() == 1); - PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0))); + std::string_view blob_data = blob_array.GetView(0); + // Only a data-evolution partial-update write interprets the sentinel, which marks a row + // the update did not touch; any other write stores the bytes verbatim. See + // BlobDefs::kPlaceholderSentinel for the accepted collision with a sentinel-equal user + // value. + if (write_placeholder_ && BlobDefs::IsPlaceholderSentinel(blob_data.data(), blob_data.size())) { + bin_lengths_.push_back(BlobDefs::kPlaceholderBinLength); + return Status::OK(); + } + PAIMON_RETURN_NOT_OK(WriteBlob(blob_data)); PAIMON_RETURN_NOT_OK(Flush()); return Status::OK(); } diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 9dbe57a87..6111168dc 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -68,9 +68,15 @@ class BlobFormatWriter : public FormatWriter { /// `write_null_on_missing_file` is enabled; otherwise a missing file follows /// `write_null_on_fetch_failure` like any other failed open. /// See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. + /// + /// `write_placeholder` (see BlobDefs::kWritePlaceholderKey, false unless the write is a + /// data-evolution partial update) persists a value exactly equal to + /// BlobDefs::kPlaceholderSentinel as a placeholder entry (bin_length -2, no data bytes); + /// any other value is stored verbatim. When disabled, values are never interpreted and can + /// never be turned into placeholder entries. static Result> Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -91,7 +97,7 @@ class BlobFormatWriter : public FormatWriter { BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, bool write_null_on_fetch_failure, - const std::shared_ptr& fs, + bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); @@ -135,6 +141,7 @@ class BlobFormatWriter : public FormatWriter { std::shared_ptr metrics_; bool write_null_on_missing_file_ = false; bool write_null_on_fetch_failure_ = false; + bool write_placeholder_ = false; uint64_t null_on_missing_file_count_ = 0; uint64_t null_on_fetch_failure_count_ = 0; std::unique_ptr logger_; diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index d54f6042c..74b5b3b6c 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -129,7 +129,16 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> CreateDefaultWriter() const { return BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_); + } + + /// Create a writer in placeholder mode, as used by data-evolution partial updates. + Result> CreatePlaceholderWriter() const { + return BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/true, file_system_, pool_); } Status AddBatchOnce(const std::shared_ptr& format_writer, @@ -144,8 +153,7 @@ class BlobFormatWriterTestBase : public ::testing::Test { return paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, pool_); } - /// Build a single-row blob array holding `bytes` verbatim, for bytes no Blob can produce, - /// such as a truncated descriptor. + /// Build a single-row blob array holding `bytes` verbatim, bypassing the Blob helpers. Result> MakeBlobArrayFromBytes(const std::string& bytes) const { arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), {std::make_shared()}); @@ -161,9 +169,11 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> ReadBackAsData() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, - /*blob_as_descriptor=*/false, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); @@ -253,7 +263,8 @@ TEST_P(BlobFormatWriterTest, TestSimple) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -284,41 +295,47 @@ TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(nullptr, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. out is nullptr"); // Test with nullptr data type ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, nullptr, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, nullptr), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, nullptr), "blob format writer create failed. pool is nullptr"); // Test with nullptr file system ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, nullptr, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, nullptr, pool_), "blob format writer create failed. fs is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, multi_field_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, non_blob_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "field regular_col: binary is not BLOB"); } @@ -443,7 +460,8 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -492,7 +510,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -525,7 +544,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, @@ -563,7 +583,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); std::string file = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, @@ -602,7 +623,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); // Row 0: missing file -> NULL. ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, @@ -667,7 +689,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), 0); } @@ -676,7 +699,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); } // The same fetch failure, now converted to NULL. @@ -684,7 +708,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); } // Missing file with only fetch-failure enabled: no existence check runs, so the file is @@ -695,7 +720,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), open_calls_before + 1); ASSERT_OK_AND_ASSIGN( @@ -713,7 +739,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); // One check before the open and one after it. ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); @@ -733,7 +760,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); ASSERT_OK_AND_ASSIGN( @@ -752,7 +780,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -762,7 +791,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -785,14 +815,16 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnInvalidDescriptor) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "invalid blob descriptor"); } { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -820,7 +852,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 1); ASSERT_OK(writer->Flush()); @@ -850,7 +883,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, exists_fail_fs, pool_)); // The reported failure names the check and keeps the underlying status message. Status check_status = AddBatchOnce(writer, array); ASSERT_NOK_WITH_MSG(check_status, "failed to check existence of blob file"); @@ -864,7 +898,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); // One check before the open and one after it failed. ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 2); @@ -884,7 +919,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( side_stream, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 0); ASSERT_OK_AND_ASSIGN( @@ -931,4 +967,238 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_EQ(buffer, expected); } +/// Placeholder tests always feed the sentinel bytes of the placeholder write protocol, so +/// they do not depend on the blob_as_descriptor_ parameter and run once on the +/// non-parameterized fixture. +using BlobFormatWriterPlaceholderTest = BlobFormatWriterTestBase; + +std::string PlaceholderSentinelBytes() { + return std::string(BlobDefs::PlaceholderSentinelView()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestWritePlaceholderGoldenBytes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + + // row 0: inline bytes "inline"; row 1: null; row 2: placeholder + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(blob_builder->AppendNull().ok()); + std::shared_ptr null_array; + ASSERT_TRUE(struct_builder.Finish(&null_array).ok()); + ASSERT_OK(AddBatchOnce(writer, null_array)); + + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + // Verify byte-level alignment with the Java writer (BlobFormatWriterTest + // testRawBlobGoldenBytes): null and placeholder rows occupy no data bytes; the index + // records [22, -1, -2] as zigzag varint deltas [0x2c, 0x2d, 0x01]. + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_TRUE(input_stream); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); + ASSERT_EQ(file_length, 30); + std::vector buffer(file_length); + ASSERT_OK_AND_ASSIGN(auto read_length, + input_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); + ASSERT_EQ(read_length, 30); + std::vector expected = {{// record 0: magic + "inline" + bin_length(22) + crc32 + 0xcf, 0x11, 0x4e, 0x58, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x60, + 0xc8, 0xe9, + // index of [22, -1, -2] + 0x2c, 0x2d, 0x01, + // footer: index length + version + 0x03, 0x00, 0x00, 0x00, 0x01}}; + ASSERT_EQ(buffer, expected); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderStrictAndAwareModes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + + // default (strict) mode: reading a placeholder entry fails + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "placeholder"); + } + + // placeholder-aware mode: the entry is returned as the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), "inline"); + ASSERT_FALSE(binary_array->IsNull(1)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } + + // placeholder-aware descriptor mode also returns the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderWithSelectionBitmap) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto array0, MakeBlobArrayFromBytes("first")); + ASSERT_OK(AddBatchOnce(writer, array0)); + ASSERT_OK_AND_ASSIGN(auto array1, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, array1)); + ASSERT_OK_AND_ASSIGN(auto array2, MakeBlobArrayFromBytes("third")); + ASSERT_OK(AddBatchOnce(writer, array2)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + RoaringBitmap32 selection; + selection.Add(1); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, selection)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); + ASSERT_EQ(binary_array->GetString(1), "third"); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelBytesVerbatimWithoutPlaceholderMode) { + // outside placeholder mode a user blob whose bytes equal the sentinel is a normal value: it + // must be stored as a real entry (not persisted as bin_length -2) and read back unchanged + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); + ASSERT_OK_AND_ASSIGN(auto sentinel_array, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, sentinel_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 1); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_FALSE(binary_array->IsNull(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelPrefixedValueVerbatimInPlaceholderMode) { + // placeholders are identified by exact equality only: even in placeholder mode a real + // value that merely starts with the sentinel bytes is stored verbatim and read back + // unchanged in both strict and placeholder-aware modes + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + std::string sentinel = PlaceholderSentinelBytes(); + ASSERT_OK_AND_ASSIGN(auto doubled_sentinel_array, MakeBlobArrayFromBytes(sentinel + sentinel)); + ASSERT_OK(AddBatchOnce(writer, doubled_sentinel_array)); + ASSERT_OK_AND_ASSIGN(auto prefixed_array, MakeBlobArrayFromBytes(sentinel + "suffix")); + ASSERT_OK(AddBatchOnce(writer, prefixed_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + for (bool emit_placeholder_sentinel : {false, true}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + emit_placeholder_sentinel, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(struct_array->length(), 2); + ASSERT_EQ(binary_array->GetString(0), sentinel + sentinel); + ASSERT_EQ(binary_array->GetString(1), sentinel + "suffix"); + } +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/blob/blob_reader_builder.h b/src/paimon/format/blob/blob_reader_builder.h index eb2ce9522..a5d20fbdd 100644 --- a/src/paimon/format/blob/blob_reader_builder.h +++ b/src/paimon/format/blob/blob_reader_builder.h @@ -43,7 +43,11 @@ class BlobReaderBuilder : public ReaderBuilder { PAIMON_ASSIGN_OR_RAISE( bool blob_as_descriptor, OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false)); - return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, pool_); + PAIMON_ASSIGN_OR_RAISE(bool emit_placeholder_sentinel, + OptionsUtils::GetValueFromMap( + options_, BlobDefs::kEmitPlaceholderSentinelKey, false)); + return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, + emit_placeholder_sentinel, pool_); } private: diff --git a/src/paimon/format/blob/blob_stats_extractor.cpp b/src/paimon/format/blob/blob_stats_extractor.cpp index ca5efe831..e840012e7 100644 --- a/src/paimon/format/blob/blob_stats_extractor.cpp +++ b/src/paimon/format/blob/blob_stats_extractor.cpp @@ -48,10 +48,13 @@ BlobStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_ fmt::format("field {} is not BLOB", write_schema_->field(0)->ToString())); } + // The reader only serves footer metadata (GetNumberOfRows); NextBatch is never called, so + // the strict placeholder mode is irrelevant even for files containing placeholder entries. PAIMON_ASSIGN_OR_RAISE( std::unique_ptr blob_reader, BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1024, /*blob_as_descriptor=*/true, pool)); + /*batch_size=*/1024, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool)); ColumnStatsVector result_stats; result_stats.push_back( ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt)); diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index 12caa0e16..7cc0b5927 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -24,6 +24,7 @@ #include #include "arrow/api.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/utils/options_utils.h" #include "paimon/defs.h" #include "paimon/format/blob/blob_format_writer.h" @@ -73,8 +74,11 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { PAIMON_ASSIGN_OR_RAISE(bool write_null_on_fetch_failure, OptionsUtils::GetValueFromMap( options_, Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, false)); + PAIMON_ASSIGN_OR_RAISE( + bool write_placeholder, + OptionsUtils::GetValueFromMap(options_, BlobDefs::kWritePlaceholderKey, false)); return BlobFormatWriter::Create(out, data_type_, write_null_on_missing_file, - write_null_on_fetch_failure, fs_, pool_); + write_null_on_fetch_failure, write_placeholder, fs_, pool_); } private: diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 398482dae..560f77217 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -42,6 +42,7 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" @@ -1101,6 +1102,595 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) { ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); } +/// Build a single-blob-column StructArray for a data-evolution partial update: "PH" marks a row +/// whose blob is not updated (persisted as a placeholder entry), std::nullopt a null blob. +std::shared_ptr MakeBlobUpdateArray( + const std::shared_ptr& blob_field, + const std::vector>& rows) { + auto struct_type = arrow::struct_({blob_field}); + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string_view sentinel = BlobDefs::PlaceholderSentinelView(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return std::dynamic_pointer_cast(array); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { + // the blob column is updated to null below, so it must be nullable + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Initial full-row write assigns row ids 0-2. + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "blob_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // Partial update: only row 1 gets a new blob, the untouched rows are written as + // placeholder entries and must fall back to the previous blob file when read. + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // updating a blob to null is not a placeholder: the null must win over older layers + auto null_update_array = MakeBlobUpdateArray(fields[2], {std::nullopt, "PH", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {}, {"b0"}, {null_update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array2)); + + // row ids still come from the data files and stay aligned with the fallback result + auto expected_with_row_id = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}), + R"([ + [1, "a", null, 0], + [2, "b", "updated_b", 1], + [3, "c", "blob_c", 2] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); + + // blob_as_descriptor read mode: the fallback-merged values come back as descriptors and + // must still resolve to the same bytes + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto desc_result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(desc_result.chunked_array); + auto desc_concat = arrow::Concatenate(desc_result.chunked_array->chunks()).ValueOrDie(); + auto desc_struct = std::dynamic_pointer_cast(desc_concat); + ASSERT_TRUE(desc_struct); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(desc_struct, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array2)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << "\nexpected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "blob_3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // second layer updates row 0 within rows [0, 1] + auto update_array1 = MakeBlobUpdateArray(fields[2], {"update1_0", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // third layer updates row 3 within rows [2, 3]; each layer only partially covers the + // range, the uncovered parts behave as placeholders + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "update2_3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "update1_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "update2_3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // row-range pushdown where the selected rows fall in one layer's file and in another + // layer's uncovered gap: row 1 is a gap row for the third layer, row 2 for the second + auto expected_middle = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [2, "b", "blob_1"], + [3, "c", "blob_2"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_middle, + /*predicate=*/nullptr, /*row_ranges=*/{Range(1, 2)})); + + // per-row reads resolve every row independently + const std::vector expected_rows = { + R"([[1, "a", "update1_0"]])", R"([[2, "b", "blob_1"]])", R"([[3, "c", "blob_2"]])", + R"([[4, "d", "update2_3"]])"}; + for (int32_t i = 0; i < static_cast(expected_rows.size()); i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_rows[i]) + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Mirrors the layer shapes of Java's BlobUpdateTest.testReadCompactedBlobSequenceGroups + // (with a single base write, as the C++ write path assigns one sequence per commit): + // row id: 0 1 2 3 4 5 6 7 8 9 + // seq1: [b0 b1 b2 b3 b4 b5 b6 b7 b8 b9] + // seq2: [u20 P P u23 u24] . . . . . + // seq3: . . . . . [P u46 P u48 P] + // seq4: [P u61 P P P P P P P u69] + // result: u20 u61 b2 u23 u24 b5 u46 b7 u48 u69 + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array1 = MakeBlobUpdateArray(fields[2], {"u20", "PH", "PH", "u23", "u24"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "u46", "PH", "u48", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/5, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto update_array3 = MakeBlobUpdateArray( + fields[2], {"PH", "u61", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "u69"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs3, WriteArray(table_path, {}, {"b0"}, {update_array3})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs3); + ASSERT_OK(Commit(table_path, commit_msgs3)); + + const std::vector expected_blobs = {"u20", "u61", "blob_2", "u23", "u24", + "blob_5", "u46", "blob_7", "u48", "u69"}; + auto expected_row = [&](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"" + expected_blobs[i] + + "\""; + }; + auto expected_full = PrepareBulkData(10, expected_row, fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); + + // every row resolves independently under row-range pushdown + for (int32_t i = 0; i < 10; i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + "[[" + expected_row(i) + "]]") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // partial update touching rows 1 and 9 only + auto update_array = MakeBlobUpdateArray( + fields[2], {"PH", "update_1", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "update_9"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // push down row ranges hitting updated and untouched rows + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "name_1", "update_1"], + [5, "name_5", "blob_5"], + [9, "name_9", "update_9"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 1), Range(5, 5), Range(9, 9)})); + + // full read still resolves every row + auto expected_full = PrepareBulkData( + 10, + [](int32_t i) { + std::string blob = (i == 1 || i == 9) ? "\"update_" + std::to_string(i) + "\"" + : "\"blob_" + std::to_string(i) + "\""; + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", " + blob; + }, + fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); +} + +/// A BLOB sequence layer whose physical file covers only a strict subrange of the full row id +/// range: the newest group is internally Gap(2), File([u2, u3]), Gap(6). The table needs a +/// normal column to be creatable, but only the blob column is ever written, so the split +/// contains only blob files and the blob bunch itself carries the row-tracking fields: the +/// fallback reader must keep _ROW_ID correct and report each row's _SEQUENCE_NUMBER from the +/// layer that resolved it, without ever exposing the internal placeholder sentinel. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // old layer: rows 0-9 all real + std::vector> base_rows; + for (int32_t i = 0; i < 10; i++) { + base_rows.emplace_back("b" + std::to_string(i)); + } + auto base_array = MakeBlobUpdateArray(fields[1], base_rows); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // new layer: first_row_id=2, row_count=2, covering only rows 2-3 + auto update_array = MakeBlobUpdateArray(fields[1], {"u2", "u3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 10); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + int64_t old_layer_seq = seq_col->Value(0); + int64_t new_layer_seq = seq_col->Value(2); + ASSERT_GT(new_layer_seq, old_layer_seq); + for (int64_t i = 0; i < 10; i++) { + ASSERT_FALSE(blob_col->IsNull(i)); + std::string expected_blob = + (i == 2 || i == 3) ? "u" + std::to_string(i) : "b" + std::to_string(i); + // the leading and trailing gaps never expose the internal placeholder sentinel + ASSERT_EQ(blob_col->GetString(i), expected_blob) << "row " << i; + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_EQ(seq_col->Value(i), (i == 2 || i == 3) ? new_layer_seq : old_layer_seq) + << "row " << i; + } + + // row-range pushdown with the row-tracking projection: the selection removes rows inside + // the blob files, so batch positions must still map back to the right file row indexes + ASSERT_OK_AND_ASSIGN(auto range_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"}, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 2), Range(8, 8)})); + ASSERT_TRUE(range_read.chunked_array); + auto range_concat = arrow::Concatenate(range_read.chunked_array->chunks()).ValueOrDie(); + auto range_struct = std::dynamic_pointer_cast(range_concat); + ASSERT_TRUE(range_struct); + ASSERT_EQ(range_struct->length(), 3); + auto range_blob_col = + std::dynamic_pointer_cast(range_struct->GetFieldByName("b0")); + auto range_row_id_col = std::dynamic_pointer_cast( + range_struct->GetFieldByName(SpecialFields::RowId().Name())); + auto range_seq_col = std::dynamic_pointer_cast( + range_struct->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(range_blob_col && range_row_id_col && range_seq_col); + const std::vector expected_row_ids = {1, 2, 8}; + const std::vector expected_blobs = {"b1", "u2", "b8"}; + for (int64_t i = 0; i < 3; i++) { + ASSERT_EQ(range_blob_col->GetString(i), expected_blobs[i]) << "row " << i; + ASSERT_EQ(range_row_id_col->Value(i), expected_row_ids[i]) << "row " << i; + ASSERT_EQ(range_seq_col->Value(i), expected_row_ids[i] == 2 ? new_layer_seq : old_layer_seq) + << "row " << i; + } +} + +/// A row that is a placeholder in every BLOB layer degrades to a null blob but keeps its +/// _ROW_ID and reports -1 as its _SEQUENCE_NUMBER. Only the blob column is ever written, so +/// the blob bunch itself carries the row-tracking fields. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // the base layer itself holds a placeholder at row 1, so after the second layer also + // leaves it untouched, row 1 is a placeholder in every layer + auto base_array = MakeBlobUpdateArray(fields[1], {"blob_a", "PH", "blob_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array = MakeBlobUpdateArray(fields[1], {"PH", "PH", "update_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 3); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + + ASSERT_EQ(blob_col->GetString(0), "blob_a"); + ASSERT_EQ(blob_col->GetString(2), "update_c"); + // the all-placeholder row: null blob, row id kept, sequence number -1 + ASSERT_TRUE(blob_col->IsNull(1)); + ASSERT_EQ(row_id_col->Value(1), 1); + ASSERT_EQ(seq_col->Value(1), -1); + for (int64_t i : {static_cast(0), static_cast(2)}) { + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_GE(seq_col->Value(i), 0); + } + ASSERT_GT(seq_col->Value(2), seq_col->Value(0)); +} + +/// A normal (full-row) write never interprets blob bytes: a user value whose bytes exactly +/// equal the placeholder sentinel must be stored verbatim (not persisted as a bin_length -2 +/// entry) and read back unchanged. The sentinel is reserved only inside the data-evolution +/// partial-update channels (see BlobDefs::kPlaceholderSentinel). +TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // full-row write: row 0's blob is exactly the sentinel bytes + auto struct_type = arrow::struct_(fields); + arrow::StructBuilder struct_builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(), std::make_shared(), + std::make_shared()}); + auto f0_builder = static_cast(struct_builder.field_builder(0)); + auto f1_builder = static_cast(struct_builder.field_builder(1)); + auto b0_builder = static_cast(struct_builder.field_builder(2)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(1).ok()); + ASSERT_TRUE(f1_builder->Append("a").ok()); + ASSERT_TRUE(b0_builder->Append(sentinel_bytes.data(), sentinel_bytes.size()).ok()); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(2).ok()); + ASSERT_TRUE(f1_builder->Append("b").ok()); + ASSERT_TRUE(b0_builder->Append("normal", 6).ok()); + std::shared_ptr src_array; + ASSERT_TRUE(struct_builder.Finish(&src_array).ok()); + auto src_struct = std::dynamic_pointer_cast(src_array); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_struct})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // read back unchanged: the sentinel-equal bytes are a normal value + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_struct)); + + // a value that merely starts with the sentinel bytes stays a normal value even through a + // partial-update fallback (placeholders are identified by exact equality only) + std::string prefixed_bytes = sentinel_bytes + "suffix"; + auto update_array = MakeBlobUpdateArray(fields[2], {prefixed_bytes, "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, ScanAndReadResult(table_path, schema->field_names())); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_result = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_result); + ASSERT_EQ(struct_result->length(), 2); + auto blob_col = + std::dynamic_pointer_cast(struct_result->GetFieldByName("b0")); + ASSERT_TRUE(blob_col); + ASSERT_FALSE(blob_col->IsNull(0)); + ASSERT_EQ(blob_col->GetString(0), prefixed_bytes); + ASSERT_EQ(blob_col->GetString(1), "updated_b"); +} + +TEST_P(BlobTableInteTest, TestBlobSentinelValueInBaseLayerDegradesToNull) { + // Pins the accepted collision of the byte-identified placeholder protocol (see + // BlobDefs::kPlaceholderSentinel): the fallback merge byte-compares every layer, so a user + // blob equal to the sentinel bytes that a later partial update leaves untouched reads as a + // placeholder in every layer and degrades to a null blob instead of falling back. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // the full-row write stores row 0's sentinel-equal bytes verbatim (the write channel is off) + std::string src_json = + std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])"; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // the partial update leaves row 0 untouched, so its base value collides with the + // placeholder markers of the newer layer + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // row 0's non-blob fields survive (they come from the data file); only the blob degrades + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); +} + +TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) { + // blob.internal.* options are reserved for the data-evolution write and read paths. Set in + // the user table options they must be ignored: were the write key honored, this full-row + // write would persist the sentinel-equal user value as a placeholder entry that no older + // layer can resolve, making the table unreadable. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {BlobDefs::kWritePlaceholderKey, "true"}, + {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + std::string src_json = + std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])"; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {src_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // the sentinel-equal value round-trips verbatim: the user-supplied keys were stripped + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_array)); +} + TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};