diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index df911dc7ce5da4..eeff6c1ca8ba2b 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -48,9 +48,21 @@ jobs: - 'be/**' - 'gensrc/proto/**' - 'gensrc/thrift/**' + arrow_paimon_changes: + - '.github/workflows/be-ut-mac.yml' + - 'thirdparty/arrow-paimon-vars.sh' + - 'thirdparty/vars.sh' + - 'thirdparty/download-thirdparty.sh' + - 'thirdparty/build-thirdparty.sh' + - 'thirdparty/paimon-cpp-cache.cmake' + - 'thirdparty/patches/apache-arrow-*.patch' + - 'thirdparty/patches/paimon-cpp-*.patch' - name: Ccache ${{ github.ref }} - if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }} + if: | + github.event_name == 'schedule' || + steps.filter.outputs.be_changes == 'true' || + steps.filter.outputs.arrow_paimon_changes == 'true' uses: ./.github/actions/ccache-action with: key: BE-UT-macOS @@ -58,7 +70,10 @@ jobs: restore-keys: BE-UT-macOS- - name: Build BE ${{ github.ref }} - if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }} + if: | + github.event_name == 'schedule' || + steps.filter.outputs.be_changes == 'true' || + steps.filter.outputs.arrow_paimon_changes == 'true' run: | cellars=( 'm4' @@ -86,6 +101,16 @@ jobs: ) brew install "${cellars[@]}" || true + # Paimon's bundled dependencies still require pre-CMake-4 policy + # compatibility. Match the supported version used by the full + # thirdparty jobs instead of relying on Homebrew's latest CMake. + brew unlink cmake || true + wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz + tar -xzf cmake-3.25.3-macos-universal.tar.gz + sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/ + export CUSTOM_CMAKE=/usr/local/bin/cmake + "${CUSTOM_CMAKE}" --version + # macos-15 runners are Apple Silicon (arm64), so download the arm64 # prebuilt thirdparty. Using the x86_64 archive makes the linker ignore # every thirdparty static library ("found architecture 'x86_64', @@ -101,6 +126,19 @@ jobs: -o doris-thirdparty-prebuilt-darwin-arm64.tar.xz fi tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz + + # Rebuild the Arrow/Paimon stack when its dedicated inputs or shared + # download/build entry points change. + # The shared source bundle can predate vars in the PR, so explicitly + # fetch Arrow's bundled source dependencies before the targeted build. + if [[ "${{ steps.filter.outputs.arrow_paimon_changes }}" == "true" ]]; then + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ + -o doris-thirdparty-source.tgz + tar -zxvf doris-thirdparty-source.tgz + ./download-thirdparty.sh arrow paimon_cpp xsimd brotli + export MACOSX_DEPLOYMENT_TARGET=12.0 + ./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp + fi popd # macos-15 runners are Apple Silicon (arm64), so the JDK env var is diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 73d2be34a32750..6a4577ac6c3036 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -195,9 +195,9 @@ jobs: # Install specific version of cmake brew unlink cmake || true - wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz - tar -xzf cmake-3.22.1-macos-universal.tar.gz - sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ + wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz + tar -xzf cmake-3.25.3-macos-universal.tar.gz + sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/ cmake --version - name: Build @@ -260,9 +260,9 @@ jobs: # Install specific version of cmake brew unlink cmake || true - wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz - tar -xzf cmake-3.22.1-macos-universal.tar.gz - sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ + wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz + tar -xzf cmake-3.25.3-macos-universal.tar.gz + sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/ cmake --version - name: Build @@ -278,4 +278,3 @@ jobs: cd thirdparty #./build-thirdparty.sh -j "$(nproc)" ./build-thirdparty.sh -j 2 - diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index b1ac9e05957fcb..aed51c2fb2e644 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -666,7 +666,7 @@ endif() set(PAIMON_FACTORY_REGISTRY_LIBS) if (ENABLE_PAIMON_CPP) # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via + # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). # No dual-stack selection needed — single Arrow for everything. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 92dffd98da875b..dde5292096adaf 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -105,6 +105,7 @@ add_thirdparty(brotlidec LIB64) add_thirdparty(brotlienc LIB64) add_thirdparty(zstd LIB64) add_thirdparty(arrow LIB64) +add_thirdparty(arrow_compute LIB64) add_thirdparty(arrow_flight LIB64) add_thirdparty(arrow_flight_sql LIB64) add_thirdparty(arrow_dataset LIB64) diff --git a/be/src/format/table/iceberg/arrow_schema_util.cpp b/be/src/format/table/iceberg/arrow_schema_util.cpp index e7186e6a48618f..5515d5c43cf7eb 100644 --- a/be/src/format/table/iceberg/arrow_schema_util.cpp +++ b/be/src/format/table/iceberg/arrow_schema_util.cpp @@ -97,7 +97,7 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field, case iceberg::TypeID::DECIMAL: { auto* dt = dynamic_cast(field.field_type()); - arrow_type = arrow::decimal(dt->get_precision(), dt->get_scale()); + arrow_type = arrow::decimal128(dt->get_precision(), dt->get_scale()); break; } diff --git a/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp b/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp index b8005c168d1f4d..34eebda2afe570 100644 --- a/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_decimal_test.cpp @@ -309,7 +309,7 @@ TEST_F(DataTypeDecimalSerDeTest, serdes) { // Run with UBSan enabled to catch misalignment errors. TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) { // 1.Prepare the data. - arrow::Decimal128Builder builder(arrow::decimal(38, 30)); + arrow::Decimal128Builder builder(arrow::decimal128(38, 30)); std::vector decimal_strings = {"12345.67", "89.10", "1112.13", "1415.16", "1718.19"}; @@ -335,7 +335,7 @@ TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) { // 4. Create Arrow array with unaligned memory auto unaligned_buffer = arrow::Buffer::Wrap(unaligned_data, num_elements * element_size); - auto arr = std::make_shared(arrow::decimal(38, 30), num_elements, + auto arr = std::make_shared(arrow::decimal128(38, 30), num_elements, unaligned_buffer); const auto* raw_values_ptr = arr->raw_values(); @@ -432,7 +432,7 @@ uint8_t* find_misaligned_address(std::vector& storage) { } TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAlignedDecimalV2TypeAlignment) { - arrow::Decimal128Builder builder(arrow::decimal(27, 9)); + arrow::Decimal128Builder builder(arrow::decimal128(27, 9)); std::vector decimal_strings = {"1.000000000", "-123456789012345678.123456789"}; for (const auto& str : decimal_strings) { @@ -453,7 +453,7 @@ TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAlignedDecimalV2TypeAlignment) { memcpy(unaligned_data, original_data, num_elements * element_size); auto unaligned_buffer = arrow::Buffer::Wrap(unaligned_data, num_elements * element_size); - auto arr = std::make_shared(arrow::decimal(27, 9), num_elements, + auto arr = std::make_shared(arrow::decimal128(27, 9), num_elements, unaligned_buffer); const auto* raw_values_ptr = arr->raw_values(); diff --git a/be/test/format_v2/table/remote_doris_reader_test.cpp b/be/test/format_v2/table/remote_doris_reader_test.cpp index a8affbde3117a3..ad8149c9d05bfc 100644 --- a/be/test/format_v2/table/remote_doris_reader_test.cpp +++ b/be/test/format_v2/table/remote_doris_reader_test.cpp @@ -213,7 +213,7 @@ TFileRangeDesc remote_doris_range(const BlockingFlightServer& server) { auto range = remote_doris_range(); auto& params = range.table_format_params.remote_doris_params; params.__set_location_uri("grpc://localhost:" + std::to_string(server.port())); - arrow::flight::Ticket ticket {.ticket = "ticket"}; + arrow::flight::Ticket ticket {"ticket"}; params.__set_ticket(ticket.SerializeToString().ValueOrDie()); return range; } diff --git a/docker/compilation/Dockerfile.gcc10 b/docker/compilation/Dockerfile.gcc10 index 29318343512fa7..30d82867ef62e0 100644 --- a/docker/compilation/Dockerfile.gcc10 +++ b/docker/compilation/Dockerfile.gcc10 @@ -26,7 +26,7 @@ RUN yum makecache && yum -y update && yum -y groupinstall 'Development Tools' && devtoolset-10-libasan-devel # build cmake -ARG CMAKE_VERSION=3.22.1 +ARG CMAKE_VERSION=3.25.3 ARG CMAKE_BASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION} RUN wget ${CMAKE_BASE_URL}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh -q -O /tmp/cmake-install.sh && \ chmod u+x /tmp/cmake-install.sh && \ @@ -121,4 +121,3 @@ ENV JAVA_HOME="/usr/lib/jvm/java-11" \ WORKDIR /root CMD ["/bin/bash"] - diff --git a/docker/compilation/arm/Dockerfile b/docker/compilation/arm/Dockerfile index 895d8ac224fcec..d52fe6be0255d1 100644 --- a/docker/compilation/arm/Dockerfile +++ b/docker/compilation/arm/Dockerfile @@ -47,10 +47,21 @@ RUN wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.9.1/ && sh /tmp/ldb_toolchain_gen.aarch64.sh /var/local/ldb-toolchain/ \ && rm /tmp/ldb_toolchain_gen.aarch64.sh +# Arrow 24 requires CMake 3.25 or newer. The bundled ldb toolchain only +# provides CMake 3.22, so install and select a compatible native build. +ARG CMAKE_VERSION=3.25.3 +ARG CMAKE_BASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION} +RUN wget ${CMAKE_BASE_URL}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && /tmp/cmake-install.sh --skip-license --prefix=/usr --exclude-subdir \ + && rm /tmp/cmake-install.sh + # there is a repo which is included all of thirdparty ENV REPOSITORY_URL="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/" \ DEFAULT_DIR="/var/local" \ JAVA_HOME="/usr/lib/jvm/java-11" \ + CUSTOM_CMAKE="/usr/bin/cmake" \ PATH="/var/local/ldb-toolchain/bin/:$PATH" # disable auto enable ccache @@ -76,6 +87,7 @@ ENV JAVA_HOME="/usr/lib/jvm/java-11" \ MAVEN_HOME="/usr/share/maven" \ REPOSITORY_URL="" \ DEFAULT_DIR="/var/local" \ + CUSTOM_CMAKE="/usr/bin/cmake" \ PATH="/var/local/ldb-toolchain/bin/:/var/local/thirdparty/installed/bin/:$PATH" \ DORIS_THIRDPARTY="/var/local/thirdparty" WORKDIR /root diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out index 48765f6c4ddaeb..0094b0f105f0c5 100644 --- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_stats2.out @@ -7,7 +7,7 @@ true 11 111 1.1 1.1 1111 1234.5678 1234.567890 123456789012345678.123456789012 a 0 PARQUET 2 {1:2, 2:2, 3:2, 4:2, 5:2, 6:2, 7:2, 8:2, 9:2, 10:2, 11:2, 12:2} {1:0, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0, 10:0, 11:0, 12:0} {1:0x00, 2:0x0B000000, 3:0x6F00000000000000, 4:0xCDCC8C3F, 5:0x9A9999999999F13F, 6:0x00000457, 7:0x00BC614E, 8:0x00000000499602D2, 9:0x000000018EE90FF6C373E0393713FA14, 10:0x616161, 11:0x9E4B0000, 12:0x005CE70F33F10500} {1:0x01, 2:0x16000000, 3:0xDE00000000000000, 4:0xCDCC0C40, 5:0x9A99999999990140, 6:0x000008AE, 7:0x05397FB1, 8:0x000000020A75E124, 9:0x0000000C7748819DFFB62505316873CB, 10:0x626262, 11:0x434C0000, 12:0x40D91FA833FE0500} -- !sql_2 -- -{"bigint_col":{"column_size":118, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":49, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":94, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":118, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01 20:34:56.000000", "upper_bound":"2023-06-16 07:45:01.000000"}, "decimal_col1":{"column_size":94, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":94, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":118, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.567890, "upper_bound":8765.432100}, "decimal_col4":{"column_size":166, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":118, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":94, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":94, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":86, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} +{"bigint_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":111, "upper_bound":222}, "boolean_col":{"column_size":33, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":0, "upper_bound":1}, "date_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01", "upper_bound":"2023-06-15"}, "datetime_col1":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"2023-01-01 20:34:56.000000", "upper_bound":"2023-06-16 07:45:01.000000"}, "decimal_col1":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1111, "upper_bound":2222}, "decimal_col2":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.5678, "upper_bound":8765.4321}, "decimal_col3":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1234.567890, "upper_bound":8765.432100}, "decimal_col4":{"column_size":90, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":123456789012345678.123456789012, "upper_bound":987654321098765432.987654321099}, "double_col":{"column_size":74, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "float_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":1.1, "upper_bound":2.2}, "int_col":{"column_size":66, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":11, "upper_bound":22}, "string_col":{"column_size":72, "value_count":2, "null_value_count":0, "nan_value_count":null, "lower_bound":"aaa", "upper_bound":"bbb"}} -- !sql_3 -- false 22 222 2.2 2.2 2222 8765.4321 8765.432100 987654321098765432.987654321099 bbb 2023-06-15 2023-06-15T23:45:01 diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh new file mode 100644 index 00000000000000..3e52968a53c06c --- /dev/null +++ b/thirdparty/arrow-paimon-vars.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# shellcheck disable=2034 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can +# distinguish this stack from unrelated thirdparty changes. + +# arrow +ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz" +ARROW_NAME="apache-arrow-24.0.0.tar.gz" +ARROW_SOURCE="arrow-apache-arrow-24.0.0" +ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" + +# Arrow bundled dependencies +BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" +BROTLI_NAME="brotli-1.0.9.tar.gz" +BROTLI_SOURCE="brotli-1.0.9" +BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" + +XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz" +XSIMD_NAME="14.0.0.tar.gz" +XSIMD_SOURCE=xsimd-14.0.0 +XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" + +# paimon-cpp +PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" +PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" +PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" +PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 2a468540484fd3..524bee50a40ce1 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1088,9 +1088,7 @@ build_arrow() { ldflags="-L${TP_LIB_DIR}" fi - CPPFLAGS="-I${TP_INCLUDE_DIR}" \ - CXXFLAGS="-I${TP_INCLUDE_DIR}" \ - LDFLAGS="${ldflags}" \ + LDFLAGS="${ldflags}" \ "${CMAKE_CMD}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_BUILD_SHARED=OFF \ @@ -1125,6 +1123,7 @@ build_arrow() { -Dxsimd_SOURCE=BUNDLED \ -DBrotli_SOURCE=BUNDLED \ -DARROW_LZ4_USE_SHARED=OFF \ + -DLZ4_ROOT="${TP_INSTALL_DIR};${TP_INSTALL_DIR}/include/lz4" \ -DLZ4_LIB="${TP_INSTALL_DIR}/lib/liblz4.a" -DLZ4_INCLUDE_DIR="${TP_INSTALL_DIR}/include/lz4" \ -DLz4_SOURCE=SYSTEM \ -DARROW_ZSTD_USE_SHARED=OFF \ @@ -1147,6 +1146,7 @@ build_arrow() { cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${TP_INSTALL_DIR}/lib64/libbrotlidec.a" cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${TP_INSTALL_DIR}/lib64/libbrotlicommon.a" strip_lib libarrow.a + strip_lib libarrow_compute.a strip_lib libparquet.a strip_lib libarrow_dataset.a strip_lib libarrow_acero.a @@ -2091,6 +2091,7 @@ build_paimon_cpp() { mkdir -p "${paimon_deps_dir}" for paimon_arrow_dep in \ libarrow.a \ + libarrow_compute.a \ libarrow_filesystem.a \ libarrow_dataset.a \ libarrow_acero.a \ diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index b4d3e5b42d744c..03d7bd2f74c210 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -438,19 +438,19 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then fi cd - fi - if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-17.0.0" ]]; then + if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-24.0.0" ]]; then cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then # Paimon-cpp parquet patches: row-group-aware batch reader, max_row_group_size, # GetBufferedSize(), int96 NANO guard, and Thrift_VERSION empty fix. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-paimon.patch" - # apache-arrow-17.0.0-force-write-int96-timestamps.patch : - # Introducing the parameter that forces writing int96 timestampes for compatibility with Paimon cpp. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-force-write-int96-timestamps.patch" + # Introducing the parameter that forces writing INT96 timestamps for + # compatibility with the Doris Parquet writer. + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-force-write-int96-timestamps.patch" # Add Parquet LZO page decompression support used by file scanner v2. - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-lzo.patch" + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-24.0.0-lzo.patch" touch "${PATCHED_MARK}" fi cd - @@ -732,13 +732,28 @@ fi # patch paimon-cpp if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - if patch -p1 -N --batch --dry-run <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" >/dev/null 2>&1; then - patch -p1 -N --batch <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" + PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" + if [[ ! -f "${PAIMON_CPP_ARROW_24_PATCHED_MARK}" ]]; then + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" + touch "${PATCHED_MARK}" + fi + patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compatibility.patch" + touch "${PAIMON_CPP_ARROW_24_PATCHED_MARK}" + fi + PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK="patched_mark_arrow_24_compute" + if [[ ! -f "${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" ]]; then + if patch -p1 --batch --forward --dry-run \ + <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compute.patch" >/dev/null; then + patch -p1 --batch --forward <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compute.patch" else - echo "Skip paimon-cpp patch: already applied or not applicable for current source" + # An earlier Arrow 24 patch set embedded this delta in the generic + # patch. Assert that exact migration state instead of silently + # accepting a partially patched source tree. + patch -p1 --batch --reverse --dry-run \ + <"${TP_PATCH_DIR}/paimon-cpp-arrow-24-compute.patch" >/dev/null fi - touch "${PATCHED_MARK}" + touch "${PAIMON_CPP_ARROW_24_COMPUTE_PATCHED_MARK}" fi cd - echo "Finished patching ${PAIMON_CPP_SOURCE}" diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake index dbebd94a0ccf63..6ab6049c59b0d5 100644 --- a/thirdparty/paimon-cpp-cache.cmake +++ b/thirdparty/paimon-cpp-cache.cmake @@ -65,7 +65,7 @@ set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") # ============================================================================ # Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) -# Doris's Arrow 17.0.0 is built with the full module set that paimon-cpp +# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp # needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). # ============================================================================ set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") @@ -74,6 +74,7 @@ set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 dire set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") +set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") diff --git a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch similarity index 74% rename from thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch rename to thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch index 5a75424756671d..1847a50694f014 100644 --- a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-force-write-int96-timestamps.patch @@ -1,20 +1,20 @@ -diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc ---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:23:23.651831424 +0800 -+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:28:36.855281965 +0800 -@@ -178,7 +178,8 @@ +--- a/cpp/src/parquet/arrow/schema.cc ++++ b/cpp/src/parquet/arrow/schema.cc +@@ -215,8 +215,9 @@ // The user is explicitly asking for Impala int96 encoding, there is no // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { +- if (arrow_properties.support_deprecated_int96_timestamps() && +- target_unit == ::arrow::TimeUnit::NANO) { + if (arrow_properties.force_write_int96_timestamps() || -+ (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO)) { ++ (arrow_properties.support_deprecated_int96_timestamps() && ++ target_unit == ::arrow::TimeUnit::NANO)) { *physical_type = ParquetType::INT96; return Status::OK(); } -diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h ---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h 2026-03-27 01:23:23.643831362 +0800 -+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h 2026-03-27 01:27:47.717897537 +0800 -@@ -980,6 +980,7 @@ +--- a/cpp/src/parquet/properties.h ++++ b/cpp/src/parquet/properties.h +@@ -1309,6 +1309,7 @@ public: Builder() : write_timestamps_as_int96_(false), @@ -22,7 +22,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar coerce_timestamps_enabled_(false), coerce_timestamps_unit_(::arrow::TimeUnit::SECOND), truncated_timestamps_allowed_(false), -@@ -1005,6 +1006,21 @@ +@@ -1334,6 +1335,21 @@ return this; } @@ -44,7 +44,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar /// \brief Coerce all timestamps to the specified time unit. /// \param unit time unit to truncate to. /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds. -@@ -1085,7 +1101,8 @@ +@@ -1423,7 +1439,8 @@ /// Create the final properties. std::shared_ptr build() { return std::shared_ptr(new ArrowWriterProperties( @@ -52,9 +52,9 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar + write_timestamps_as_int96_, force_write_int96_timestamps_, + coerce_timestamps_enabled_, coerce_timestamps_unit_, truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, - engine_version_, use_threads_, executor_)); + engine_version_, use_threads_, executor_, write_time_adjusted_to_utc_)); } -@@ -1093,6 +1110,8 @@ +@@ -1431,6 +1448,8 @@ private: bool write_timestamps_as_int96_; @@ -63,7 +63,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar bool coerce_timestamps_enabled_; ::arrow::TimeUnit::type coerce_timestamps_unit_; bool truncated_timestamps_allowed_; -@@ -1107,6 +1126,8 @@ +@@ -1447,6 +1466,8 @@ bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; } @@ -72,7 +72,7 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; } ::arrow::TimeUnit::type coerce_timestamps_unit() const { return coerce_timestamps_unit_; -@@ -1138,6 +1159,7 @@ +@@ -1483,6 +1504,7 @@ private: explicit ArrowWriterProperties(bool write_nanos_as_int96, @@ -80,16 +80,16 @@ diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h ar bool coerce_timestamps_enabled, ::arrow::TimeUnit::type coerce_timestamps_unit, bool truncated_timestamps_allowed, bool store_schema, -@@ -1145,6 +1167,7 @@ - EngineVersion engine_version, bool use_threads, - ::arrow::internal::Executor* executor) +@@ -1491,6 +1513,7 @@ + ::arrow::internal::Executor* executor, + bool write_time_adjusted_to_utc) : write_timestamps_as_int96_(write_nanos_as_int96), + force_write_int96_timestamps_(force_write_int96_timestamps), coerce_timestamps_enabled_(coerce_timestamps_enabled), coerce_timestamps_unit_(coerce_timestamps_unit), truncated_timestamps_allowed_(truncated_timestamps_allowed), -@@ -1155,6 +1178,7 @@ - executor_(executor) {} +@@ -1502,6 +1525,7 @@ + write_time_adjusted_to_utc_(write_time_adjusted_to_utc) {} const bool write_timestamps_as_int96_; + const bool force_write_int96_timestamps_; diff --git a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch b/thirdparty/patches/apache-arrow-24.0.0-lzo.patch similarity index 55% rename from thirdparty/patches/apache-arrow-17.0.0-lzo.patch rename to thirdparty/patches/apache-arrow-24.0.0-lzo.patch index a983818413a01c..d7076509756f3f 100644 --- a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-lzo.patch @@ -1,11 +1,26 @@ --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc -@@ -30,0 +31,2 @@ -+ +@@ -29,6 +29,8 @@ + #include + #include + +#include -@@ -268,0 +269 @@ ++ + #include "arrow/array.h" + #include "arrow/array/array_binary.h" + #include "arrow/array/builder_binary.h" +@@ -228,6 +230,7 @@ + const CryptoContext* crypto_ctx, bool always_compressed) + : properties_(properties), + stream_(std::move(stream)), + compression_codec_(codec), -@@ -279 +282,7 @@ + decompression_buffer_(AllocateBuffer(properties_.memory_pool(), 0)), + page_ordinal_(0), + seen_num_values_(0), +@@ -237,7 +240,13 @@ + InitDecryption(); + } + max_page_header_size_ = kDefaultMaxPageHeaderSize; - decompressor_ = GetCodec(codec); + if (compression_codec_ == Compression::LZO) { + if (lzo_init() != LZO_E_OK) { @@ -14,12 +29,30 @@ + } else { + decompressor_ = GetCodec(codec); + } -@@ -315,0 +325 @@ + always_compressed_ = always_compressed; + } + +@@ -271,6 +280,7 @@ + format::PageHeader current_page_header_; + + // Compression codec to use. + Compression::type compression_codec_; -@@ -585 +595 @@ + std::unique_ptr<::arrow::util::Codec> decompressor_; + std::shared_ptr decompression_buffer_; + +@@ -548,7 +558,7 @@ + std::shared_ptr SerializedPageReader::DecompressIfNeeded( + std::shared_ptr page_buffer, int compressed_len, int uncompressed_len, + int levels_byte_len) { - if (decompressor_ == nullptr) { + if (decompressor_ == nullptr && compression_codec_ != Compression::LZO) { -@@ -601,0 +612,61 @@ + return page_buffer; + } + if (compressed_len < levels_byte_len || uncompressed_len < levels_byte_len) { +@@ -565,6 +575,74 @@ + memcpy(decompressed, page_buffer->data(), levels_byte_len); + } + + if (compression_codec_ == Compression::LZO) { + const uint8_t* input = page_buffer->data() + levels_byte_len; + const uint8_t* const input_end = page_buffer->data() + compressed_len; @@ -34,7 +67,8 @@ + + while (input < input_end) { + if (input_end - input < 4) { -+ throw ParquetException("LZO page decompression failed: truncated large block length"); ++ throw ParquetException( ++ "LZO page decompression failed: truncated large block length"); + } + + uint32_t large_block_uncompressed_len = load_big_endian_u32(input); @@ -45,13 +79,15 @@ + + while (large_block_uncompressed_len > 0) { + if (input_end - input < 4) { -+ throw ParquetException("LZO page decompression failed: truncated small block length"); ++ throw ParquetException( ++ "LZO page decompression failed: truncated small block length"); + } + + uint32_t small_block_compressed_len = load_big_endian_u32(input); + input += 4; + if (static_cast(input_end - input) < small_block_compressed_len) { -+ throw ParquetException("LZO page decompression failed: truncated small block data"); ++ throw ParquetException( ++ "LZO page decompression failed: truncated small block data"); + } + + auto small_block_uncompressed_len = @@ -64,7 +100,8 @@ + std::to_string(result)); + } + if (small_block_uncompressed_len > large_block_uncompressed_len) { -+ throw ParquetException("LZO page decompression failed: invalid small block size"); ++ throw ParquetException( ++ "LZO page decompression failed: invalid small block size"); + } + + input += small_block_compressed_len; @@ -73,12 +110,17 @@ + } + } + if (output != output_end) { -+ throw ParquetException("Page didn't decompress to expected size, expected: " + -+ std::to_string(uncompressed_len - levels_byte_len) + ", but got:" + -+ std::to_string(output - (decompression_buffer_->mutable_data() + -+ levels_byte_len))); ++ throw ParquetException( ++ "Page didn't decompress to expected size, expected: " + ++ std::to_string(uncompressed_len - levels_byte_len) + ++ ", but got:" + ++ std::to_string( ++ output - (decompression_buffer_->mutable_data() + levels_byte_len))); + } + + return decompression_buffer_; + } + + // GH-31992: DataPageV2 may store only levels and no values when all + // values are null. In this case, Parquet java is known to produce a + // 0-len compressed area (which is invalid compressed input). diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch b/thirdparty/patches/apache-arrow-24.0.0-paimon.patch similarity index 68% rename from thirdparty/patches/apache-arrow-17.0.0-paimon.patch rename to thirdparty/patches/apache-arrow-24.0.0-paimon.patch index 4e53117b79b65b..583f85b4400bbf 100644 --- a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch +++ b/thirdparty/patches/apache-arrow-24.0.0-paimon.patch @@ -1,23 +1,19 @@ -diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc -index ec3890a41f..943f69bb6c 100644 --- a/cpp/src/parquet/arrow/schema.cc +++ b/cpp/src/parquet/arrow/schema.cc -@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, +@@ -215,7 +215,8 @@ // The user is explicitly asking for Impala int96 encoding, there is no // logical type. - if (arrow_properties.support_deprecated_int96_timestamps()) { -+ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { ++ if (arrow_properties.support_deprecated_int96_timestamps() && ++ target_unit == ::arrow::TimeUnit::NANO) { *physical_type = ParquetType::INT96; return Status::OK(); } - -diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc -index 285e2a5973..aa6f92f077 100644 --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc -@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector& row_groups, - return Status::OK(); +@@ -1041,25 +1041,32 @@ + ::arrow::MakeVectorIterator(std::move(batches)), std::move(batch_schema)); } - int64_t num_rows = 0; @@ -28,7 +24,7 @@ index 285e2a5973..aa6f92f077 100644 } using ::arrow::RecordBatchIterator; -+ int row_group_idx = 0; ++ size_t row_group_idx = 0; // NB: This lambda will be invoked outside the scope of this call to // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. @@ -40,26 +36,23 @@ index 285e2a5973..aa6f92f077 100644 this]() mutable -> ::arrow::Result { ::arrow::ChunkedArrayVector columns(readers.size()); -- // don't reserve more rows than necessary + // don't reserve more rows than necessary - int64_t batch_size = std::min(properties().batch_size(), num_rows); - num_rows -= batch_size; + int64_t batch_size = 0; + if (!num_rows.empty()) { -+ // don't reserve more rows than necessary + batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); + num_rows[row_group_idx] -= batch_size; -+ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) != row_group_idx) { -+ row_group_idx++; ++ if (num_rows[row_group_idx] == 0 && row_group_idx + 1 < num_rows.size()) { ++ ++row_group_idx; + } + } RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( reader_properties_.use_threads(), static_cast(readers.size()), -diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc -index 4fd7ef1b47..87326a54f1 100644 --- a/cpp/src/parquet/arrow/writer.cc +++ b/cpp/src/parquet/arrow/writer.cc -@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter { +@@ -341,6 +341,14 @@ return Status::OK(); } @@ -68,13 +61,13 @@ index 4fd7ef1b47..87326a54f1 100644 + return 0; + } + return row_group_writer_->total_compressed_bytes() + -+ row_group_writer_->total_compressed_bytes_written(); ++ row_group_writer_->total_compressed_bytes_written(); + } + Status Close() override { if (!closed_) { // Make idempotent -@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter { +@@ -444,10 +452,14 @@ // Max number of rows allowed in a row group. const int64_t max_row_group_length = this->properties().max_row_group_length(); @@ -84,16 +77,15 @@ index 4fd7ef1b47..87326a54f1 100644 if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || - row_group_writer_->num_rows() >= max_row_group_length) { + row_group_writer_->num_rows() >= max_row_group_length || -+ (row_group_writer_->total_compressed_bytes_written() + -+ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) { ++ row_group_writer_->total_compressed_bytes_written() + ++ row_group_writer_->total_compressed_bytes() >= ++ max_row_group_size) { RETURN_NOT_OK(NewBufferedRowGroup()); } -diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h -index 4a1a033a7b..0f13d05e44 100644 --- a/cpp/src/parquet/arrow/writer.h +++ b/cpp/src/parquet/arrow/writer.h -@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter { +@@ -124,6 +124,9 @@ /// option in this case. virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; @@ -103,11 +95,9 @@ index 4a1a033a7b..0f13d05e44 100644 /// \brief Write the footer and close the file. virtual ::arrow::Status Close() = 0; virtual ~FileWriter(); -diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h -index 4d3acb491e..3906ff3c59 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h -@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true; +@@ -160,6 +160,7 @@ static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; @@ -115,28 +105,28 @@ index 4d3acb491e..3906ff3c59 100644 static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; -@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties { +@@ -343,6 +344,7 @@ dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), + max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), pagesize_(kDefaultDataPageSize), + max_rows_per_page_(kDefaultMaxRowsPerPage), version_(ParquetVersion::PARQUET_2_6), - data_page_version_(ParquetDataPageVersion::V1), -@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties { +@@ -359,6 +361,7 @@ dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), write_batch_size_(properties.write_batch_size()), max_row_group_length_(properties.max_row_group_length()), + max_row_group_size_(properties.max_row_group_size()), pagesize_(properties.data_pagesize()), + max_rows_per_page_(properties.max_rows_per_page()), version_(properties.version()), - data_page_version_(properties.data_page_version()), -@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties { +@@ -468,6 +471,13 @@ return this; } + /// Specify the max bytes size to put in a single row group. -+ /// Default 128 M. ++ /// Default 128MB. + Builder* max_row_group_size(int64_t max_row_group_size) { + max_row_group_size_ = max_row_group_size; + return this; @@ -145,24 +135,33 @@ index 4d3acb491e..3906ff3c59 100644 /// Specify the data page size. /// Default 1MB. Builder* data_pagesize(int64_t pg_size) { -@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties { +@@ -867,11 +877,12 @@ return std::shared_ptr(new WriterProperties( pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, -- pagesize_, version_, created_by_, page_checksum_enabled_, -+ max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_, - std::move(file_encryption_properties_), default_column_properties_, - column_properties, data_page_version_, store_decimal_as_integer_, - std::move(sorting_columns_))); -@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties { +- pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, +- size_statistics_level_, std::move(file_encryption_properties_), +- default_column_properties_, column_properties, data_page_version_, +- store_decimal_as_integer_, std::move(sorting_columns_), +- content_defined_chunking_enabled_, content_defined_chunking_options_)); ++ max_row_group_size_, pagesize_, max_rows_per_page_, version_, created_by_, ++ page_checksum_enabled_, size_statistics_level_, ++ std::move(file_encryption_properties_), default_column_properties_, ++ column_properties, data_page_version_, store_decimal_as_integer_, ++ std::move(sorting_columns_), content_defined_chunking_enabled_, ++ content_defined_chunking_options_)); + } + + private: +@@ -881,6 +892,7 @@ int64_t dictionary_pagesize_limit_; int64_t write_batch_size_; int64_t max_row_group_length_; + int64_t max_row_group_size_; int64_t pagesize_; + int64_t max_rows_per_page_; ParquetVersion::type version_; - ParquetDataPageVersion data_page_version_; -@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties { +@@ -917,6 +929,8 @@ inline int64_t max_row_group_length() const { return max_row_group_length_; } @@ -170,40 +169,43 @@ index 4d3acb491e..3906ff3c59 100644 + inline int64_t data_pagesize() const { return pagesize_; } - inline ParquetDataPageVersion data_page_version() const { -@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties { + inline int64_t max_rows_per_page() const { return max_rows_per_page_; } +@@ -1046,9 +1060,10 @@ private: explicit WriterProperties( MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, -- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version, -+ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version, - const std::string& created_by, bool page_write_checksum_enabled, +- int64_t max_row_group_length, int64_t pagesize, int64_t max_rows_per_page, +- ParquetVersion::type version, const std::string& created_by, +- bool page_write_checksum_enabled, SizeStatisticsLevel size_statistics_level, ++ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ++ int64_t max_rows_per_page, ParquetVersion::type version, ++ const std::string& created_by, bool page_write_checksum_enabled, ++ SizeStatisticsLevel size_statistics_level, std::shared_ptr file_encryption_properties, const ColumnProperties& default_column_properties, -@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties { + const std::unordered_map& column_properties, +@@ -1059,6 +1074,7 @@ dictionary_pagesize_limit_(dictionary_pagesize_limit), write_batch_size_(write_batch_size), max_row_group_length_(max_row_group_length), + max_row_group_size_(max_row_group_size), pagesize_(pagesize), + max_rows_per_page_(max_rows_per_page), parquet_data_page_version_(data_page_version), - parquet_version_(version), -@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties { +@@ -1078,6 +1094,7 @@ int64_t dictionary_pagesize_limit_; int64_t write_batch_size_; int64_t max_row_group_length_; + int64_t max_row_group_size_; int64_t pagesize_; + int64_t max_rows_per_page_; ParquetDataPageVersion parquet_data_page_version_; - ParquetVersion::type parquet_version_; -diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -index 9df922afa2..5c8b3d4d07 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) - REQUIRED_VERSION - 0.11.0) - +@@ -1884,7 +1884,20 @@ + REQUIRED_VERSION + 0.11.0) + - string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) + if(NOT Thrift_VERSION) + if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch b/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch new file mode 100644 index 00000000000000..d0e723a34d98ef --- /dev/null +++ b/thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch @@ -0,0 +1,64 @@ +diff --git a/include/paimon/status.h b/include/paimon/status.h +--- a/include/paimon/status.h ++++ b/include/paimon/status.h +@@ -458,9 +458,4 @@ + } \ + } while (false) + +-// This is an internal-use macro and should not be used in public headers. +-#ifndef RETURN_NOT_OK +-#define RETURN_NOT_OK(s) PAIMON_RETURN_NOT_OK(s) +-#endif +- + } // namespace paimon +diff --git a/src/paimon/format/parquet/parquet_input_stream_impl.cpp b/src/paimon/format/parquet/parquet_input_stream_impl.cpp +--- a/src/paimon/format/parquet/parquet_input_stream_impl.cpp ++++ b/src/paimon/format/parquet/parquet_input_stream_impl.cpp +@@ -59,7 +59,7 @@ arrow::Result> ParquetInputStreamImpl::Read(int64 + arrow::AllocateResizableBuffer(nbytes, pool_.get())); + ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, Read(nbytes, buffer->mutable_data())); + if (read_bytes < nbytes) { +- RETURN_NOT_OK(buffer->Resize(read_bytes)); ++ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); + } + return std::shared_ptr(std::move(buffer)); + } +@@ -78,7 +78,7 @@ arrow::Result> ParquetInputStreamImpl::ReadAt(int + arrow::AllocateResizableBuffer(nbytes, pool_.get())); + ARROW_ASSIGN_OR_RAISE(int64_t read_bytes, ReadAt(position, nbytes, buffer->mutable_data())); + if (read_bytes < nbytes) { +- RETURN_NOT_OK(buffer->Resize(read_bytes)); ++ ARROW_RETURN_NOT_OK(buffer->Resize(read_bytes)); + } + return std::shared_ptr(std::move(buffer)); + } +diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp +--- a/src/paimon/format/parquet/file_reader_wrapper.cpp ++++ b/src/paimon/format/parquet/file_reader_wrapper.cpp +@@ -81,8 +81,9 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { + PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(target_row_groups_[j])); + target_row_group_indices.push_back(row_group_id); + } +- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( +- target_row_group_indices, target_column_indices_, &batch_reader_)); ++ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( ++ batch_reader_, file_reader_->GetRecordBatchReader(target_row_group_indices, ++ target_column_indices_)); + return Status::OK(); + } + } +@@ -152,9 +153,11 @@ Status FileReaderWrapper::PrepareForReading( + std::vector> target_row_groups; + PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); + std::unique_ptr batch_reader; +- PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( +- std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), +- column_indices, &batch_reader)); ++ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( ++ batch_reader, ++ file_reader_->GetRecordBatchReader( ++ std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), ++ column_indices)); + target_row_groups_ = target_row_groups; + target_column_indices_ = column_indices; + batch_reader_ = std::move(batch_reader); diff --git a/thirdparty/patches/paimon-cpp-arrow-24-compute.patch b/thirdparty/patches/paimon-cpp-arrow-24-compute.patch new file mode 100644 index 00000000000000..1b581b8f93ee92 --- /dev/null +++ b/thirdparty/patches/paimon-cpp-arrow-24-compute.patch @@ -0,0 +1,44 @@ +diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake +--- a/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cmake_modules/ThirdpartyToolchain.cmake +@@ -38,6 +38,8 @@ + set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH + "Include directory for external Arrow/Parquet headers") + set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") ++set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "" CACHE FILEPATH ++ "Path to external libarrow_compute.a") + set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") + set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") + set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") +@@ -1008,6 +1010,7 @@ + + foreach(_paimon_external_lib + IN ITEMS PAIMON_EXTERNAL_ARROW_LIB ++ PAIMON_EXTERNAL_ARROW_COMPUTE_LIB + PAIMON_EXTERNAL_ARROW_DATASET_LIB + PAIMON_EXTERNAL_ARROW_ACERO_LIB + PAIMON_EXTERNAL_PARQUET_LIB +@@ -1028,6 +1031,13 @@ + INTERFACE_INCLUDE_DIRECTORIES + "${ARROW_INCLUDE_DIR}") + ++ add_library(arrow_compute STATIC IMPORTED) ++ set_target_properties(arrow_compute ++ PROPERTIES IMPORTED_LOCATION ++ "${PAIMON_EXTERNAL_ARROW_COMPUTE_LIB}" ++ INTERFACE_INCLUDE_DIRECTORIES ++ "${ARROW_INCLUDE_DIR}") ++ + add_library(arrow_dataset STATIC IMPORTED) + set_target_properties(arrow_dataset + PROPERTIES IMPORTED_LOCATION +@@ -1055,7 +1065,9 @@ + INTERFACE_INCLUDE_DIRECTORIES + "${ARROW_INCLUDE_DIR}") + +- target_link_libraries(arrow_acero INTERFACE arrow) ++ target_link_libraries(arrow_compute INTERFACE arrow) ++ ++ target_link_libraries(arrow_acero INTERFACE arrow_compute) + + target_link_libraries(arrow_dataset INTERFACE arrow_acero) diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index f8d18a3a2bea16..bfffb61f91a57f 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -50,6 +50,10 @@ export TP_JAR_DIR="${TP_INSTALL_DIR}/lib/jar" # export REPOSITORY_URL= DORIS_THIRDPARTY_REPOSITORY_URL="${DORIS_THIRDPARTY_REPOSITORY_URL:-https://doris-regression-hk.oss-cn-hongkong.aliyuncs.com/regression/datalake/thirdparty/juicefs}" +DORIS_THIRDPARTY_VARS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=arrow-paimon-vars.sh +. "${DORIS_THIRDPARTY_VARS_DIR}/arrow-paimon-vars.sh" + ##################################################### # Download url, filename and unpaced filename # of all thirdparties @@ -235,12 +239,6 @@ ZSTD_NAME=zstd-1.5.7.tar.gz ZSTD_SOURCE=zstd-1.5.7 ZSTD_MD5SUM="780fc1896922b1bc52a4e90980cdda48" -# brotli -BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz" -BROTLI_NAME="brotli-1.0.9.tar.gz" -BROTLI_SOURCE="brotli-1.0.9" -BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d" - # flatbuffers FLATBUFFERS_DOWNLOAD="https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz" FLATBUFFERS_NAME=flatbuffers-23.5.26.tar.gz @@ -260,14 +258,6 @@ GRPC_NAME="grpc-v1.54.3.tar.gz" GRPC_SOURCE=grpc-1.54.3 GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de" -# arrow -# Arrow 19.0.1 will MacOS compile error and decimal type error when convert to Parquet. -# https://github.com/apache/doris/pull/51217 -ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-17.0.0.tar.gz" -ARROW_NAME="apache-arrow-17.0.0.tar.gz" -ARROW_SOURCE="arrow-apache-arrow-17.0.0" -ARROW_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0" - # Abseil ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/releases/download/20250512.1/abseil-cpp-20250512.1.tar.gz" ABSEIL_NAME="abseil-cpp-20250512.1.tar.gz" @@ -427,13 +417,6 @@ BENCHMARK_NAME=benchmark-v1.8.0.tar.gz BENCHMARK_SOURCE=benchmark-1.8.0 BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817" -# xsimd -# for arrow-17.0.0, if arrow upgrade, this version may also need to be changed -XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz" -XSIMD_NAME="13.0.0.tar.gz" -XSIMD_SOURCE=xsimd-13.0.0 -XSIMD_MD5SUM="c661deb91836e82d3070f81032014fe6" - # simdjson SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.11.6.tar.gz" SIMDJSON_NAME=simdjson-3.11.6.tar.gz @@ -567,12 +550,6 @@ PUGIXML_NAME=pugixml-1.15.tar.gz PUGIXML_SOURCE=pugixml-1.15 PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895" -# paimon-cpp -PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" -PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" - # lance-c LANCE_C_DOWNLOAD="https://github.com/lance-format/lance-c/archive/refs/tags/v0.1.2.tar.gz" LANCE_C_NAME="lance-c-v0.1.2.tar.gz"