From 3b3d7b6d815f5bb715f51305a1d2c4ef93a5f832 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 28 Jul 2026 23:09:22 +0800 Subject: [PATCH 1/2] [chore](thirdparty) Upgrade Apache Arrow to 24.0.0 ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Doris used Apache Arrow 17.0.0, which blocked consuming newer Arrow fixes and APIs. Upgrade the C++ third-party dependency to 24.0.0, align xsimd with Arrow 24, port the Paimon row-group, INT96, and Parquet LZO compatibility patches, add Arrow Compute to the static dependency graph, and migrate removed Arrow and Parquet APIs while preserving Decimal128 semantics. ### Release note None ### Check List (For Author) - Test: Manual test - Built the Arrow third-party dependency - Built the Paimon C++ third-party dependency from clean sources - Built BE with `./build.sh --be` - Verified the Arrow and Paimon patches apply cleanly from pristine sources - Ran `build-support/check-format.sh` - Relevant C++ unit test source was updated but not built or run - Behavior changed: No - Does this need documentation: No --- be/CMakeLists.txt | 2 +- be/cmake/thirdparty.cmake | 1 + .../table/iceberg/arrow_schema_util.cpp | 2 +- .../data_type_serde_decimal_test.cpp | 8 +- thirdparty/build-thirdparty.sh | 7 +- thirdparty/download-thirdparty.sh | 22 +-- thirdparty/paimon-cpp-cache.cmake | 3 +- ...24.0.0-force-write-int96-timestamps.patch} | 42 ++--- ...zo.patch => apache-arrow-24.0.0-lzo.patch} | 72 +++++++-- ...patch => apache-arrow-24.0.0-paimon.patch} | 146 +++++++++--------- .../paimon-cpp-arrow-24-compatibility.patch | 64 ++++++++ .../paimon-cpp-buildutils-static-deps.patch | 18 ++- thirdparty/vars.sh | 19 +-- 13 files changed, 263 insertions(+), 143 deletions(-) rename thirdparty/patches/{apache-arrow-17.0.0-force-write-int96-timestamps.patch => apache-arrow-24.0.0-force-write-int96-timestamps.patch} (74%) rename thirdparty/patches/{apache-arrow-17.0.0-lzo.patch => apache-arrow-24.0.0-lzo.patch} (55%) rename thirdparty/patches/{apache-arrow-17.0.0-paimon.patch => apache-arrow-24.0.0-paimon.patch} (68%) create mode 100644 thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index b1ac9e05957fcb..aed51c2fb2e644 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -666,7 +666,7 @@ endif() set(PAIMON_FACTORY_REGISTRY_LIBS) if (ENABLE_PAIMON_CPP) # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via + # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). # No dual-stack selection needed — single Arrow for everything. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 92dffd98da875b..dde5292096adaf 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -105,6 +105,7 @@ add_thirdparty(brotlidec LIB64) add_thirdparty(brotlienc LIB64) add_thirdparty(zstd LIB64) add_thirdparty(arrow LIB64) +add_thirdparty(arrow_compute LIB64) add_thirdparty(arrow_flight LIB64) add_thirdparty(arrow_flight_sql LIB64) add_thirdparty(arrow_dataset LIB64) diff --git a/be/src/format/table/iceberg/arrow_schema_util.cpp b/be/src/format/table/iceberg/arrow_schema_util.cpp index e7186e6a48618f..5515d5c43cf7eb 100644 --- a/be/src/format/table/iceberg/arrow_schema_util.cpp +++ b/be/src/format/table/iceberg/arrow_schema_util.cpp @@ -97,7 +97,7 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field, case iceberg::TypeID::DECIMAL: { auto* dt = dynamic_cast(field.field_type()); - arrow_type = arrow::decimal(dt->get_precision(), dt->get_scale()); + arrow_type = arrow::decimal128(dt->get_precision(), dt->get_scale()); break; } diff --git a/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp b/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp index b8005c168d1f4d..34eebda2afe570 100644 --- a/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp @@ -309,7 +309,7 @@ TEST_F(DataTypeDecimalSerDeTest, serdes) { // Run with UBSan enabled to catch misalignment errors. TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) { // 1.Prepare the data. - arrow::Decimal128Builder builder(arrow::decimal(38, 30)); + arrow::Decimal128Builder builder(arrow::decimal128(38, 30)); std::vector decimal_strings = {"12345.67", "89.10", "1112.13", "1415.16", "1718.19"}; @@ -335,7 +335,7 @@ TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) { // 4. Create Arrow array with unaligned memory auto unaligned_buffer = arrow::Buffer::Wrap(unaligned_data, num_elements * element_size); - auto arr = std::make_shared(arrow::decimal(38, 30), num_elements, + auto arr = std::make_shared(arrow::decimal128(38, 30), num_elements, unaligned_buffer); const auto* raw_values_ptr = arr->raw_values(); @@ -432,7 +432,7 @@ uint8_t* find_misaligned_address(std::vector& storage) { } TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAlignedDecimalV2TypeAlignment) { - arrow::Decimal128Builder builder(arrow::decimal(27, 9)); + arrow::Decimal128Builder builder(arrow::decimal128(27, 9)); std::vector decimal_strings = {"1.000000000", "-123456789012345678.123456789"}; for (const auto& str : decimal_strings) { @@ -453,7 +453,7 @@ TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAlignedDecimalV2TypeAlignment) { memcpy(unaligned_data, original_data, num_elements * element_size); auto unaligned_buffer = arrow::Buffer::Wrap(unaligned_data, num_elements * element_size); - auto arr = std::make_shared(arrow::decimal(27, 9), num_elements, + auto arr = std::make_shared(arrow::decimal128(27, 9), num_elements, unaligned_buffer); const auto* raw_values_ptr = arr->raw_values(); diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 2a468540484fd3..524bee50a40ce1 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1088,9 +1088,7 @@ build_arrow() { ldflags="-L${TP_LIB_DIR}" fi - CPPFLAGS="-I${TP_INCLUDE_DIR}" \ - CXXFLAGS="-I${TP_INCLUDE_DIR}" \ - LDFLAGS="${ldflags}" \ + LDFLAGS="${ldflags}" \ "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_BUILD_SHARED=OFF \ @@ -1125,6 +1123,7 @@ build_arrow() { -Dxsimd_SOURCE=BUNDLED \ -DBrotli_SOURCE=BUNDLED \ -DARROW_LZ4_USE_SHARED=OFF \ + -DLZ4_ROOT="${TP_INSTALL_DIR};${TP_INSTALL_DIR}/include/lz4" \ -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \ -DLz4_SOURCE=SYSTEM \ -DARROW_ZSTD_USE_SHARED=OFF \ @@ -1147,6 +1146,7 @@ build_arrow() { cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${TP_INSTALL_DIR}/lib64/libbrotlidec.a" cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${TP_INSTALL_DIR}/lib64/libbrotlicommon.a" strip_lib libarrow.a + strip_lib libarrow_compute.a strip_lib libparquet.a strip_lib libarrow_dataset.a strip_lib libarrow_acero.a @@ -2091,6 +2091,7 @@ build_paimon_cpp() { mkdir -p "${paimon_deps_dir}" for paimon_arrow_dep in \ libarrow.a \ + libarrow_compute.a \ libarrow_filesystem.a \ libarrow_dataset.a \ libarrow_acero.a \ diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index b4d3e5b42d744c..b552e1f7459995 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -438,19 +438,19 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then fi cd - fi - if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-17.0.0" ]]; then + if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-24.0.0" ]]; then cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then # Paimon-cpp parquet patches: row-group-aware batch reader, max_row_group_size, # GetBufferedSize(), int96 NANO guard, and Thrift_VERSION empty fix. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-paimon.patch" - # apache-arrow-17.0.0-force-write-int96-timestamps.patch : - # Introducing the parameter that forces writing int96 timestampes for compatibility with Paimon cpp. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-force-write-int96-timestamps.patch" + # Introducing the parameter that forces writing INT96 timestamps for + # compatibility with the Doris Parquet writer. + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-force-write-int96-timestamps.patch" # Add Parquet LZO page decompression support used by file scanner v2. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-lzo.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-lzo.patch" touch "${PATCHED_MARK}" fi cd - @@ -733,11 +733,11 @@ fi if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - if patch -p1 -N --batch --dry-run <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" >/dev/null 2>&1; then - patch -p1 -N --batch <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" - else - echo "Skip paimon-cpp patch: already applied or not applicable for current source" - fi + for patch_file in \ + "${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" \ + "${TP_PATCH_DIR}/paimon-cpp-arrow-24-compatibility.patch"; do + patch -p1 <"${patch_file}" + done touch "${PATCHED_MARK}" fi cd - diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake index dbebd94a0ccf63..6ab6049c59b0d5 100644 --- a/thirdparty/paimon-cpp-cache.cmake +++ b/thirdparty/paimon-cpp-cache.cmake @@ -65,7 +65,7 @@ set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") # ============================================================================ # Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) -# Doris's Arrow 17.0.0 is built with the full module set that paimon-cpp +# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp # needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). # ============================================================================ set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") @@ -74,6 +74,7 @@ set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 dire set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") +set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") diff --git a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch similarity index 74% rename from thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch rename to thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch index 5a75424756671d..1847a50694f014 100644 --- a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch @@ -1,20 +1,20 @@ -diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc ---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:23:23.651831424 +0800 -+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:28:36.855281965 +0800 -@@ -178,7 +178,8 @@ +--- a/cpp/src/parquet/arrow/schema.cc ++++ b/cpp/src/parquet/arrow/schema.cc +@@ -215,8 +215,9 @@ // The user is explicitly asking for Impala int96 encoding, there is no // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { +- if (arrow_properties.support_deprecated_int96_timestamps() && +- target_unit == ::arrow::TimeUnit::NANO) { + if (arrow_properties.force_write_int96_timestamps() || -+ (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO)) { ++ (arrow_properties.support_deprecated_int96_timestamps() && ++ target_unit == ::arrow::TimeUnit::NANO)) { *physical_type = ParquetType::INT96; return Status::OK(); } -diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h ---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h 2026-03-27 01:23:23.643831362 +0800 -+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h 2026-03-27 01:27:47.717897537 +0800 -@@ -980,6 +980,7 @@ +--- a/cpp/src/parquet/properties.h ++++ b/cpp/src/parquet/properties.h +@@ -1309,6 +1309,7 @@ public: Builder() : write_timestamps_as_int96_(false), @@ -22,7 +22,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar coerce_timestamps_enabled_(false), coerce_timestamps_unit_(::arrow::TimeUnit::SECOND), truncated_timestamps_allowed_(false), -@@ -1005,6 +1006,21 @@ +@@ -1334,6 +1335,21 @@ return this; } @@ -44,7 +44,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar /// \brief Coerce all timestamps to the specified time unit. /// \param unit time unit to truncate to. /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds. -@@ -1085,7 +1101,8 @@ +@@ -1423,7 +1439,8 @@ /// Create the final properties. std::shared_ptr build() { return std::shared_ptr(new ArrowWriterProperties( @@ -52,9 +52,9 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar + write_timestamps_as_int96_, force_write_int96_timestamps_, + coerce_timestamps_enabled_, coerce_timestamps_unit_, truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, - engine_version_, use_threads_, executor_)); + engine_version_, use_threads_, executor_, write_time_adjusted_to_utc_)); } -@@ -1093,6 +1110,8 @@ +@@ -1431,6 +1448,8 @@ private: bool write_timestamps_as_int96_; @@ -63,7 +63,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar bool coerce_timestamps_enabled_; ::arrow::TimeUnit::type coerce_timestamps_unit_; bool truncated_timestamps_allowed_; -@@ -1107,6 +1126,8 @@ +@@ -1447,6 +1466,8 @@ bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; } @@ -72,7 +72,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; } ::arrow::TimeUnit::type coerce_timestamps_unit() const { return coerce_timestamps_unit_; -@@ -1138,6 +1159,7 @@ +@@ -1483,6 +1504,7 @@ private: explicit ArrowWriterProperties(bool write_nanos_as_int96, @@ -80,16 +80,16 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar bool coerce_timestamps_enabled, ::arrow::TimeUnit::type coerce_timestamps_unit, bool truncated_timestamps_allowed, bool store_schema, -@@ -1145,6 +1167,7 @@ - EngineVersion engine_version, bool use_threads, - ::arrow::internal::Executor* executor) +@@ -1491,6 +1513,7 @@ + ::arrow::internal::Executor* executor, + bool write_time_adjusted_to_utc) : write_timestamps_as_int96_(write_nanos_as_int96), + force_write_int96_timestamps_(force_write_int96_timestamps), coerce_timestamps_enabled_(coerce_timestamps_enabled), coerce_timestamps_unit_(coerce_timestamps_unit), truncated_timestamps_allowed_(truncated_timestamps_allowed), -@@ -1155,6 +1178,7 @@ - executor_(executor) {} +@@ -1502,6 +1525,7 @@ + write_time_adjusted_to_utc_(write_time_adjusted_to_utc) {} const bool write_timestamps_as_int96_; + const bool force_write_int96_timestamps_; diff --git a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch b/thirdparty/patches/apache-arrow-24.0.0-lzo.patch similarity index 55% rename from thirdparty/patches/apache-arrow-17.0.0-lzo.patch rename to thirdparty/patches/apache-arrow-24.0.0-lzo.patch index a983818413a01c..80ec4937580bca 100644 --- a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-lzo.patch @@ -1,11 +1,26 @@ --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc -@@ -30,0 +31,2 @@ -+ +@@ -29,6 +29,8 @@ + #include + #include + +#include -@@ -268,0 +269 @@ ++ + #include "arrow/array.h" + #include "arrow/array/array_binary.h" + #include "arrow/array/builder_binary.h" +@@ -228,6 +230,7 @@ + const CryptoContext* crypto_ctx, bool always_compressed) + : properties_(properties), + stream_(std::move(stream)), + compression_codec_(codec), -@@ -279 +282,7 @@ + decompression_buffer_(AllocateBuffer(properties_.memory_pool(), 0)), + page_ordinal_(0), + seen_num_values_(0), +@@ -237,7 +240,13 @@ + InitDecryption(); + } + max_page_header_size_ = kDefaultMaxPageHeaderSize; - decompressor_ = GetCodec(codec); + if (compression_codec_ == Compression::LZO) { + if (lzo_init() != LZO_E_OK) { @@ -14,12 +29,30 @@ + } else { + decompressor_ = GetCodec(codec); + } -@@ -315,0 +325 @@ + always_compressed_ = always_compressed; + } + +@@ -271,6 +280,7 @@ + format::PageHeader current_page_header_; + + // Compression codec to use. + Compression::type compression_codec_; -@@ -585 +595 @@ + std::unique_ptr<::arrow::util::Codec> decompressor_; + std::shared_ptr decompression_buffer_; + +@@ -548,7 +558,7 @@ + std::shared_ptr SerializedPageReader::DecompressIfNeeded( + std::shared_ptr page_buffer, int compressed_len, int uncompressed_len, + int levels_byte_len) { - if (decompressor_ == nullptr) { + if (decompressor_ == nullptr && compression_codec_ != Compression::LZO) { -@@ -601,0 +612,61 @@ + return page_buffer; + } + if (compressed_len < levels_byte_len || uncompressed_len < levels_byte_len) { +@@ -565,6 +575,74 @@ + memcpy(decompressed, page_buffer->data(), levels_byte_len); + } + + if (compression_codec_ == Compression::LZO) { + const uint8_t* input = page_buffer->data() + levels_byte_len; + const uint8_t* const input_end = page_buffer->data() + compressed_len; @@ -34,7 +67,8 @@ + + while (input < input_end) { + if (input_end - input < 4) { -+ throw ParquetException("LZO page decompression failed: truncated large block length"); ++ throw ParquetException( ++ "LZO page decompression failed: truncated large block length"); + } + + uint32_t large_block_uncompressed_len = load_big_endian_u32(input); @@ -45,13 +79,15 @@ + + while (large_block_uncompressed_len > 0) { + if (input_end - input < 4) { -+ throw ParquetException("LZO page decompression failed: truncated small block length"); ++ throw ParquetException( ++ "LZO page decompression failed: truncated small block length"); + } + + uint32_t small_block_compressed_len = load_big_endian_u32(input); + input += 4; + if (static_cast(input_end - input) < small_block_compressed_len) { -+ throw ParquetException("LZO page decompression failed: truncated small block data"); ++ throw ParquetException( ++ "LZO page decompression failed: truncated small block data"); + } + + auto small_block_uncompressed_len = @@ -64,7 +100,8 @@ + std::to_string(result)); + } + if (small_block_uncompressed_len > large_block_uncompressed_len) { -+ throw ParquetException("LZO page decompression failed: invalid small block size"); ++ throw ParquetException( ++ "LZO page decompression failed: invalid small block size"); + } + + input += small_block_compressed_len; @@ -73,12 +110,17 @@ + } + } + if (output != output_end) { -+ throw ParquetException("Page didn't decompress to expected size, expected: " + -+ std::to_string(uncompressed_len - levels_byte_len) + ", but got:" + -+ std::to_string(output - (decompression_buffer_->mutable_data() + -+ levels_byte_len))); ++ throw ParquetException( ++ "Page didn't decompress to expected size, expected: " + ++ std::to_string(uncompressed_len - levels_byte_len) + ++ ", but got:" + ++ std::to_string( ++ output - (decompression_buffer_->mutable_data() + levels_byte_len))); + } + + return decompression_buffer_; + } + + // GH-31992: DataPageV2 may store only levels and no values when all + // values are null. In this case, Parquet java is known to produce a + // 0-len compressed area (which is invalid compressed input). diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch b/thirdparty/patches/apache-arrow-24.0.0-paimon.patch similarity index 68% rename from thirdparty/patches/apache-arrow-17.0.0-paimon.patch rename to thirdparty/patches/apache-arrow-24.0.0-paimon.patch index 4e53117b79b65b..8e7beb4d8c6991 100644 --- a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-paimon.patch @@ -1,35 +1,31 @@ -diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc -index ec3890a41f..943f69bb6c 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc -@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, - +@@ -215,7 +215,8 @@ + // The user is explicitly asking for Impala int96 encoding, there is no // logical type. - if (arrow_properties.support_deprecated_int96_timestamps()) { -+ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { ++ if (arrow_properties.support_deprecated_int96_timestamps() && ++ target_unit == ::arrow::TimeUnit::NANO) { *physical_type = ParquetType::INT96; return Status::OK(); } - -diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc -index 285e2a5973..aa6f92f077 100644 --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc -@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector& row_groups, - return Status::OK(); +@@ -1041,25 +1041,32 @@ + ::arrow::MakeVectorIterator(std::move(batches)), std::move(batch_schema)); } - + - int64_t num_rows = 0; + std::vector num_rows; for (int row_group : row_groups) { - num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); + num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows()); } - + using ::arrow::RecordBatchIterator; -+ int row_group_idx = 0; - ++ size_t row_group_idx = 0; + // NB: This lambda will be invoked outside the scope of this call to // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. // `this` is a non-owning pointer so we are relying on the parent FileReader outliving @@ -39,75 +35,69 @@ index 285e2a5973..aa6f92f077 100644 + [readers, batch_schema, num_rows, row_group_idx, this]() mutable -> ::arrow::Result { ::arrow::ChunkedArrayVector columns(readers.size()); - -- // don't reserve more rows than necessary + + // don't reserve more rows than necessary - int64_t batch_size = std::min(properties().batch_size(), num_rows); - num_rows -= batch_size; + int64_t batch_size = 0; + if (!num_rows.empty()) { -+ // don't reserve more rows than necessary + batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); + num_rows[row_group_idx] -= batch_size; -+ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) != row_group_idx) { -+ row_group_idx++; ++ if (num_rows[row_group_idx] == 0 && row_group_idx + 1 < num_rows.size()) { ++ ++row_group_idx; + } + } - + RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( reader_properties_.use_threads(), static_cast(readers.size()), -diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc -index 4fd7ef1b47..87326a54f1 100644 --- a/cpp/src/parquet/arrow/writer.cc +++ b/cpp/src/parquet/arrow/writer.cc -@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter { +@@ -341,6 +341,14 @@ return Status::OK(); } - + + int64_t GetBufferedSize() override { + if (row_group_writer_ == nullptr) { + return 0; + } + return row_group_writer_->total_compressed_bytes() + -+ row_group_writer_->total_compressed_bytes_written(); ++ row_group_writer_->total_compressed_bytes_written(); + } + Status Close() override { if (!closed_) { // Make idempotent -@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter { - +@@ -444,10 +452,14 @@ + // Max number of rows allowed in a row group. const int64_t max_row_group_length = this->properties().max_row_group_length(); + const int64_t max_row_group_size = this->properties().max_row_group_size(); - + // Initialize a new buffered row group writer if necessary. if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || - row_group_writer_->num_rows() >= max_row_group_length) { + row_group_writer_->num_rows() >= max_row_group_length || -+ (row_group_writer_->total_compressed_bytes_written() + -+ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) { ++ row_group_writer_->total_compressed_bytes_written() + ++ row_group_writer_->total_compressed_bytes() >= ++ max_row_group_size) { RETURN_NOT_OK(NewBufferedRowGroup()); } - -diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h -index 4a1a033a7b..0f13d05e44 100644 + --- a/cpp/src/parquet/arrow/writer.h +++ b/cpp/src/parquet/arrow/writer.h -@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter { +@@ -124,6 +124,9 @@ /// option in this case. virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; - + + /// \brief Return the buffered size in bytes. + virtual int64_t GetBufferedSize() = 0; + /// \brief Write the footer and close the file. virtual ::arrow::Status Close() = 0; virtual ~FileWriter(); -diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h -index 4d3acb491e..3906ff3c59 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h -@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true; +@@ -160,6 +160,7 @@ static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; @@ -115,28 +105,28 @@ index 4d3acb491e..3906ff3c59 100644 static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; -@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties { +@@ -343,6 +344,7 @@ dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), + max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), pagesize_(kDefaultDataPageSize), + max_rows_per_page_(kDefaultMaxRowsPerPage), version_(ParquetVersion::PARQUET_2_6), - data_page_version_(ParquetDataPageVersion::V1), -@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties { +@@ -359,6 +361,7 @@ dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), write_batch_size_(properties.write_batch_size()), max_row_group_length_(properties.max_row_group_length()), + max_row_group_size_(properties.max_row_group_size()), pagesize_(properties.data_pagesize()), + max_rows_per_page_(properties.max_rows_per_page()), version_(properties.version()), - data_page_version_(properties.data_page_version()), -@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties { +@@ -468,6 +471,13 @@ return this; } - + + /// Specify the max bytes size to put in a single row group. -+ /// Default 128 M. ++ /// Default 128MB. + Builder* max_row_group_size(int64_t max_row_group_size) { + max_row_group_size_ = max_row_group_size; + return this; @@ -145,64 +135,76 @@ index 4d3acb491e..3906ff3c59 100644 /// Specify the data page size. /// Default 1MB. Builder* data_pagesize(int64_t pg_size) { -@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties { - +@@ -867,11 +877,12 @@ + return std::shared_ptr(new WriterProperties( pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, -- pagesize_, version_, created_by_, page_checksum_enabled_, -+ max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_, - std::move(file_encryption_properties_), default_column_properties_, - column_properties, data_page_version_, store_decimal_as_integer_, - std::move(sorting_columns_))); -@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties { +- pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, +- size_statistics_level_, std::move(file_encryption_properties_), +- default_column_properties_, column_properties, data_page_version_, +- store_decimal_as_integer_, std::move(sorting_columns_), +- content_defined_chunking_enabled_, content_defined_chunking_options_)); ++ max_row_group_size_, pagesize_, max_rows_per_page_, version_, created_by_, ++ page_checksum_enabled_, size_statistics_level_, ++ std::move(file_encryption_properties_), default_column_properties_, ++ column_properties, data_page_version_, store_decimal_as_integer_, ++ std::move(sorting_columns_), content_defined_chunking_enabled_, ++ content_defined_chunking_options_)); + } + + private: +@@ -881,6 +892,7 @@ int64_t dictionary_pagesize_limit_; int64_t write_batch_size_; int64_t max_row_group_length_; + int64_t max_row_group_size_; int64_t pagesize_; + int64_t max_rows_per_page_; ParquetVersion::type version_; - ParquetDataPageVersion data_page_version_; -@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties { - +@@ -917,6 +929,8 @@ + inline int64_t max_row_group_length() const { return max_row_group_length_; } - + + inline int64_t max_row_group_size() const { return max_row_group_size_; } + inline int64_t data_pagesize() const { return pagesize_; } - - inline ParquetDataPageVersion data_page_version() const { -@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties { + + inline int64_t max_rows_per_page() const { return max_rows_per_page_; } +@@ -1046,9 +1060,10 @@ private: explicit WriterProperties( MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, -- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version, -+ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version, - const std::string& created_by, bool page_write_checksum_enabled, +- int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page, +- ParquetVersion::type version, const std::string& created_by, +- bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level, ++ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ++ int64_t max_rows_per_page, ParquetVersion::type version, ++ const std::string& created_by, bool page_write_checksum_enabled, ++ SizeStatisticsLevel size_statistics_level, std::shared_ptr file_encryption_properties, const ColumnProperties& default_column_properties, -@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties { + const std::unordered_map& column_properties, +@@ -1059,6 +1074,7 @@ dictionary_pagesize_limit_(dictionary_pagesize_limit), write_batch_size_(write_batch_size), max_row_group_length_(max_row_group_length), + max_row_group_size_(max_row_group_size), pagesize_(pagesize), + max_rows_per_page_(max_rows_per_page), parquet_data_page_version_(data_page_version), - parquet_version_(version), -@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties { +@@ -1078,6 +1094,7 @@ int64_t dictionary_pagesize_limit_; int64_t write_batch_size_; int64_t max_row_group_length_; + int64_t max_row_group_size_; int64_t pagesize_; + int64_t max_rows_per_page_; ParquetDataPageVersion parquet_data_page_version_; - ParquetVersion::type parquet_version_; -diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -index 9df922afa2..5c8b3d4d07 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) - REQUIRED_VERSION - 0.11.0) +@@ -1884,7 +1884,20 @@ + REQUIRED_VERSION + 0.11.0) - string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) + if(NOT Thrift_VERSION) diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch b/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch new file mode 100644 index 00000000000000..707d840cd3fb55 --- /dev/null +++ b/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch @@ -0,0 +1,64 @@ +diff --git a/include/paimon/status.h b/include/paimon/status.h +--- a/include/paimon/status.h ++++ b/include/paimon/status.h +@@ -458,9 +458,4 @@ + } \ + } while (false) + +-// This is an internal-use macro and should not be used in public headers. +-#ifndef RETURN_NOT_OK +-#define RETURN_NOT_OK(s) PAIMON_RETURN_NOT_OK(s) +-#endif +- + } // namespace paimon +diff --git a/src/paimon/format/parquet/parquet_input_stream_impl.cpp b/src/paimon/format/parquet/parquet_input_stream_impl.cpp +--- a/src/paimon/format/parquet/parquet_input_stream_impl.cpp ++++ b/src/paimon/format/parquet/parquet_input_stream_impl.cpp +@@ -59,7 +59,7 @@ arrow::Result> ParquetInputStreamImpl::Read(int64 + arrow::AllocateResizableBuffer(nbytes, pool_.get())); + ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, Read(nbytes, buffer->mutable_data())); + if (read_bytes < nbytes) { +- RETURN_NOT_OK(buffer->Resize(read_bytes)); ++ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); + } + return std::shared_ptr(std::move(buffer)); + } +@@ -78,7 +78,7 @@ arrow::Result> ParquetInputStreamImpl::ReadAt(int + arrow::AllocateResizableBuffer(nbytes, pool_.get())); + ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, ReadAt(position, nbytes, buffer->mutable_data())); + if (read_bytes < nbytes) { +- RETURN_NOT_OK(buffer->Resize(read_bytes)); ++ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); + } + return std::shared_ptr(std::move(buffer)); + } +diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp +--- a/src/paimon/format/parquet/file_reader_wrapper.cpp ++++ b/src/paimon/format/parquet/file_reader_wrapper.cpp +@@ -81,8 +81,9 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { + PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(target_row_groups_[j])); + target_row_group_indices.push_back(row_group_id); + } +- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( +- target_row_group_indices, target_column_indices_, &batch_reader_)); ++ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( ++ batch_reader_, file_reader_->GetRecordBatchReader(target_row_group_indices, ++ target_column_indices_)); + return Status::OK(); + } + } +@@ -152,9 +153,11 @@ Status FileReaderWrapper::PrepareForReading( + std::vector> target_row_groups; + PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); + std::unique_ptr batch_reader; +- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( +- std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), +- column_indices, &batch_reader)); ++ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( ++ batch_reader, ++ file_reader_->GetRecordBatchReader( ++ std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), ++ column_indices)); + target_row_groups_ = target_row_groups; + target_column_indices_ = column_indices; + batch_reader_ = std::move(batch_reader); diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch index fc1fe1553126b9..87d78029faf7b8 100644 --- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch +++ b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch @@ -114,13 +114,15 @@ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyT diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -34,6 +34,16 @@ set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" +@@ -34,6 +34,18 @@ set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") +option(PAIMON_USE_EXTERNAL_ARROW "Reuse external Arrow/Parquet instead of building arrow_ep" OFF) +set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH + "Include directory for external Arrow/Parquet headers") +set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") ++set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "" CACHE FILEPATH ++ "Path to external libarrow_compute.a") +set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") +set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") +set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") @@ -134,7 +136,7 @@ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyT diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -961,5 +961,95 @@ macro(build_orc) +@@ -961,5 +961,105 @@ macro(build_orc) endmacro() macro(build_arrow) @@ -167,6 +169,7 @@ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyT + + foreach(_paimon_external_lib + IN ITEMS PAIMON_EXTERNAL_ARROW_LIB ++ PAIMON_EXTERNAL_ARROW_COMPUTE_LIB + PAIMON_EXTERNAL_ARROW_DATASET_LIB + PAIMON_EXTERNAL_ARROW_ACERO_LIB + PAIMON_EXTERNAL_PARQUET_LIB @@ -187,6 +190,13 @@ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyT + INTERFACE_INCLUDE_DIRECTORIES + "${ARROW_INCLUDE_DIR}") + ++ add_library(arrow_compute STATIC IMPORTED) ++ set_target_properties(arrow_compute ++ PROPERTIES IMPORTED_LOCATION ++ "${PAIMON_EXTERNAL_ARROW_COMPUTE_LIB}" ++ INTERFACE_INCLUDE_DIRECTORIES ++ "${ARROW_INCLUDE_DIR}") ++ + add_library(arrow_dataset STATIC IMPORTED) + set_target_properties(arrow_dataset + PROPERTIES IMPORTED_LOCATION @@ -214,7 +224,9 @@ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyT + INTERFACE_INCLUDE_DIRECTORIES + "${ARROW_INCLUDE_DIR}") + -+ target_link_libraries(arrow_acero INTERFACE arrow) ++ target_link_libraries(arrow_compute INTERFACE arrow) ++ ++ target_link_libraries(arrow_acero INTERFACE arrow_compute) + + target_link_libraries(arrow_dataset INTERFACE arrow_acero) + diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index f8d18a3a2bea16..ae1cdba240dea4 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -261,12 +261,10 @@ GRPC_SOURCE=grpc-1.54.3 GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de" # arrow -# Arrow 19.0.1 will MacOS compile error and decimal type error when convert to Parquet. -# https://github.com/apache/doris/pull/51217 -ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-17.0.0.tar.gz" -ARROW_NAME="apache-arrow-17.0.0.tar.gz" -ARROW_SOURCE="arrow-apache-arrow-17.0.0" -ARROW_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0" +ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz" +ARROW_NAME="apache-arrow-24.0.0.tar.gz" +ARROW_SOURCE="arrow-apache-arrow-24.0.0" +ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" # Abseil ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/releases/download/20250512.1/abseil-cpp-20250512.1.tar.gz" @@ -428,11 +426,10 @@ BENCHMARK_SOURCE=benchmark-1.8.0 BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817" # xsimd -# for arrow-17.0.0, if arrow upgrade, this version may also need to be changed -XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz" -XSIMD_NAME="13.0.0.tar.gz" -XSIMD_SOURCE=xsimd-13.0.0 -XSIMD_MD5SUM="c661deb91836e82d3070f81032014fe6" +XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" +XSIMD_NAME="14.0.0.tar.gz" +XSIMD_SOURCE=xsimd-14.0.0 +XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" # simdjson SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.11.6.tar.gz" From 01bf481a849d2b00a5e5cf2f035b6e0fed313f1b Mon Sep 17 00:00:00 2001 From: daidai Date: Thu, 30 Jul 2026 10:17:26 +0800 Subject: [PATCH 2/2] [fix](build) Fix Arrow 24 CI compatibility ### What problem does this PR solve? Issue Number: None Related PR: #66221 Problem Summary: Arrow 24 requires CMake 3.25, while the macOS thirdparty workflows and gcc10 image selected older CMake releases. The macOS BE job also consumed a stale prebuilt Arrow archive, incremental Paimon sources skipped the new compatibility patch, and a BE unit test still initialized Arrow Flight Ticket as an aggregate. Raise the supported CMake floor, rebuild Arrow and Paimon for macOS thirdparty changes, version the Paimon patch marker, and use the Arrow 24 Ticket constructor. ### Release note None ### Check List (For Author) - Test: Manual test - Parsed the modified GitHub Actions workflows as YAML - Checked shell syntax and clean, incremental, and repeated Paimon patch flows - Checked the Arrow 24 Ticket constructor with a C++20 syntax compile - Ran build-support/check-format.sh - Behavior changed: No - Does this need documentation: No --- .github/workflows/be-ut-mac.yml | 23 +++++++++++++++++-- .github/workflows/build-thirdparty.yml | 13 +++++------ .../table/remote_doris_reader_test.cpp | 2 +- docker/compilation/Dockerfile.gcc10 | 3 +-- thirdparty/download-thirdparty.sh | 15 ++++++------ 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index df911dc7ce5da4..d6b7852cec29a4 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -48,9 +48,14 @@ jobs: - 'be/**' - 'gensrc/proto/**' - 'gensrc/thrift/**' + thirdparty_changes: + - 'thirdparty/**' - name: Ccache ${{ github.ref }} - if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }} + if: | + github.event_name == 'schedule' || + steps.filter.outputs.be_changes == 'true' || + steps.filter.outputs.thirdparty_changes == 'true' uses: ./.github/actions/ccache-action with: key: BE-UT-macOS @@ -58,7 +63,10 @@ jobs: restore-keys: BE-UT-macOS- - name: Build BE ${{ github.ref }} - if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }} + if: | + github.event_name == 'schedule' || + steps.filter.outputs.be_changes == 'true' || + steps.filter.outputs.thirdparty_changes == 'true' run: | cellars=( 'm4' @@ -101,6 +109,17 @@ jobs: -o doris-thirdparty-prebuilt-darwin-arm64.tar.xz fi tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz + + # A thirdparty upgrade can make the shared prebuilt archive stale. + # Rebuild the changed Arrow/Paimon stack before compiling BE so this + # job validates the versions selected by the PR. + if [[ "${{ steps.filter.outputs.thirdparty_changes }}" == "true" ]]; then + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ + -o doris-thirdparty-source.tgz + tar -zxvf doris-thirdparty-source.tgz + export MACOSX_DEPLOYMENT_TARGET=12.0 + ./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp + fi popd # macos-15 runners are Apple Silicon (arm64), so the JDK env var is diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 73d2be34a32750..6a4577ac6c3036 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -195,9 +195,9 @@ jobs: # Install specific version of cmake brew unlink cmake || true - wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz - tar -xzf cmake-3.22.1-macos-universal.tar.gz - sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ + wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz + tar -xzf cmake-3.25.3-macos-universal.tar.gz + sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/ cmake --version - name: Build @@ -260,9 +260,9 @@ jobs: # Install specific version of cmake brew unlink cmake || true - wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz - tar -xzf cmake-3.22.1-macos-universal.tar.gz - sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ + wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz + tar -xzf cmake-3.25.3-macos-universal.tar.gz + sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/ cmake --version - name: Build @@ -278,4 +278,3 @@ jobs: cd thirdparty #./build-thirdparty.sh -j "$(nproc)" ./build-thirdparty.sh -j 2 - diff --git a/be/test/format_v2/table/remote_doris_reader_test.cpp b/be/test/format_v2/table/remote_doris_reader_test.cpp index a8affbde3117a3..ad8149c9d05bfc 100644 --- a/be/test/format_v2/table/remote_doris_reader_test.cpp +++ b/be/test/format_v2/table/remote_doris_reader_test.cpp @@ -213,7 +213,7 @@ TFileRangeDesc remote_doris_range(const BlockingFlightServer& server) { auto range = remote_doris_range(); auto& params = range.table_format_params.remote_doris_params; params.__set_location_uri("grpc://localhost:" + std::to_string(server.port())); - arrow::flight::Ticket ticket {.ticket = "ticket"}; + arrow::flight::Ticket ticket {"ticket"}; params.__set_ticket(ticket.SerializeToString().ValueOrDie()); return range; } diff --git a/docker/compilation/Dockerfile.gcc10 b/docker/compilation/Dockerfile.gcc10 index 29318343512fa7..30d82867ef62e0 100644 --- a/docker/compilation/Dockerfile.gcc10 +++ b/docker/compilation/Dockerfile.gcc10 @@ -26,7 +26,7 @@ RUN yum makecache && yum -y update && yum -y groupinstall 'Development Tools' && devtoolset-10-libasan-devel # build cmake -ARG CMAKE_VERSION=3.22.1 +ARG CMAKE_VERSION=3.25.3 ARG CMAKE_BASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION} RUN wget ${CMAKE_BASE_URL}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh -q -O /tmp/cmake-install.sh && \ chmod u+x /tmp/cmake-install.sh && \ @@ -121,4 +121,3 @@ ENV JAVA_HOME="/usr/lib/jvm/java-11" \ WORKDIR /root CMD ["/bin/bash"] - diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index b552e1f7459995..c577488fac1d42 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -732,13 +732,14 @@ fi # patch paimon-cpp if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - for patch_file in \ - "${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" \ - "${TP_PATCH_DIR}/paimon-cpp-arrow-24-compatibility.patch"; do - patch -p1 <"${patch_file}" - done - touch "${PATCHED_MARK}" + PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" + if [[ ! -f "${PAIMON_CPP_ARROW_24_PATCHED_MARK}" ]]; then + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" + touch "${PATCHED_MARK}" + fi + patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compatibility.patch" + touch "${PAIMON_CPP_ARROW_24_PATCHED_MARK}" fi cd - echo "Finished patching ${PAIMON_CPP_SOURCE}"