From a2307578f5769e115b45215661610033683792a1 Mon Sep 17 00:00:00 2001 From: mwish Date: Fri, 30 Oct 2026 12:28:51 +0800 Subject: [PATCH] SIMD: optimize parquet GreaterThan syntax --- cpp/src/parquet/CMakeLists.txt | 10 ++---- cpp/src/parquet/level_comparison.cc | 7 ++-- cpp/src/parquet/level_comparison_avx2.cc | 32 ------------------- .../parquet/level_comparison_avx2_internal.h | 28 ---------------- cpp/src/parquet/level_comparison_simd_128.cc | 8 +++++ cpp/src/parquet/level_comparison_simd_256.cc | 7 ++++ .../parquet/level_comparison_simd_internal.h | 3 ++ .../level_comparison_simd_kernel_internal.h | 23 +++++++++++++ 8 files changed, 46 insertions(+), 72 deletions(-) delete mode 100644 cpp/src/parquet/level_comparison_avx2.cc delete mode 100644 cpp/src/parquet/level_comparison_avx2_internal.h diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 6affbde88b48..c37e7e605f42 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -204,11 +204,7 @@ append_runtime_avx2_src(PARQUET_SRCS level_comparison_simd_256.cc) if(ARROW_HAVE_RUNTIME_AVX2) # AVX2 is used as a proxy for BMI2. - list(APPEND - PARQUET_SRCS - level_comparison_avx2.cc - level_conversion_bmi2.cc - bloom_filter_avx2.cc) + list(APPEND PARQUET_SRCS level_conversion_bmi2.cc bloom_filter_avx2.cc) # We need CMAKE_CXX_FLAGS_RELEASE here to prevent the one-definition-rule # violation with -DCMAKE_BUILD_TYPE=MinSizeRel. CMAKE_CXX_FLAGS_RELEASE # will force inlining as much as possible. @@ -218,8 +214,8 @@ if(ARROW_HAVE_RUNTIME_AVX2) separate_arguments(RELEASE_FLAGS NATIVE_COMMAND "${CMAKE_CXX_FLAGS_RELEASE}") list(APPEND AVX2_FLAGS ${RELEASE_FLAGS}) endif() - set_source_files_properties(level_comparison_avx2.cc bloom_filter_avx2.cc - PROPERTIES COMPILE_OPTIONS "${AVX2_FLAGS}") + set_source_files_properties(bloom_filter_avx2.cc PROPERTIES COMPILE_OPTIONS + "${AVX2_FLAGS}") # WARNING: DO NOT BLINDLY COPY THIS CODE FOR OTHER BMI2 USE CASES. # This code is always guarded by runtime dispatch which verifies # BMI2 is present. For a very small number of CPUs AVX2 does not diff --git a/cpp/src/parquet/level_comparison.cc b/cpp/src/parquet/level_comparison.cc index beb9c66c5f91..c2258ccefcd8 100644 --- a/cpp/src/parquet/level_comparison.cc +++ b/cpp/src/parquet/level_comparison.cc @@ -18,11 +18,6 @@ #include #include "parquet/level_comparison.h" - -#if defined(ARROW_HAVE_RUNTIME_AVX2) -# include "parquet/level_comparison_avx2_internal.h" -#endif - #include "parquet/level_comparison_simd_internal.h" #define PARQUET_IMPL_NAMESPACE standard @@ -45,6 +40,8 @@ struct GreaterThanDynamicFunction { static constexpr auto targets() { return std::array{ ARROW_DISPATCH_TARGET_NONE(&standard::GreaterThanBitmapImpl) // + ARROW_DISPATCH_TARGET_NEON(&GreaterThanBitmapNeon) // + ARROW_DISPATCH_TARGET_SSE4_2(&GreaterThanBitmapSse42) // ARROW_DISPATCH_TARGET_AVX2(&GreaterThanBitmapAvx2) // }; } diff --git a/cpp/src/parquet/level_comparison_avx2.cc b/cpp/src/parquet/level_comparison_avx2.cc deleted file mode 100644 index 12cb122a9429..000000000000 --- a/cpp/src/parquet/level_comparison_avx2.cc +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "parquet/level_comparison_avx2_internal.h" - -#define PARQUET_IMPL_NAMESPACE avx2 -#include "parquet/level_comparison_inc.h" -#undef PARQUET_IMPL_NAMESPACE - -namespace parquet { -namespace internal { - -uint64_t GreaterThanBitmapAvx2(const int16_t* levels, int64_t num_levels, int16_t rhs) { - return avx2::GreaterThanBitmapImpl(levels, num_levels, rhs); -} - -} // namespace internal -} // namespace parquet diff --git a/cpp/src/parquet/level_comparison_avx2_internal.h b/cpp/src/parquet/level_comparison_avx2_internal.h deleted file mode 100644 index d95fb8564199..000000000000 --- a/cpp/src/parquet/level_comparison_avx2_internal.h +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "parquet/level_comparison.h" - -namespace parquet::internal { - -uint64_t GreaterThanBitmapAvx2(const int16_t* levels, int64_t num_levels, int16_t rhs); - -} // namespace parquet::internal diff --git a/cpp/src/parquet/level_comparison_simd_128.cc b/cpp/src/parquet/level_comparison_simd_128.cc index af5f5f882ad9..d35303f0f84c 100644 --- a/cpp/src/parquet/level_comparison_simd_128.cc +++ b/cpp/src/parquet/level_comparison_simd_128.cc @@ -17,9 +17,11 @@ #if defined(ARROW_HAVE_NEON) # define FIND_MIN_MAX_PLATFORM FindMinMaxNeon +# define GREATER_THAN_BITMAP_PLATFORM GreaterThanBitmapNeon # define ARCH_PLATFORM xsimd::neon64 #elif defined(ARROW_HAVE_SSE4_2) || defined(ARROW_HAVE_RUNTIME_SSE4_2) # define FIND_MIN_MAX_PLATFORM FindMinMaxSse42 +# define GREATER_THAN_BITMAP_PLATFORM GreaterThanBitmapSse42 # define ARCH_PLATFORM xsimd::sse4_2 #endif @@ -34,8 +36,14 @@ MinMax FIND_MIN_MAX_PLATFORM(const int16_t* levels, int64_t num_levels) { return FindMinMaxSimd(levels, num_levels); } +uint64_t GREATER_THAN_BITMAP_PLATFORM(const int16_t* levels, int64_t num_levels, + int16_t rhs) { + return GreaterThanBitmapSimd(levels, num_levels, rhs); +} + } // namespace parquet::internal # undef ARCH_PLATFORM +# undef GREATER_THAN_BITMAP_PLATFORM # undef FIND_MIN_MAX_PLATFORM #endif // FIND_MIN_MAX_PLATFORM diff --git a/cpp/src/parquet/level_comparison_simd_256.cc b/cpp/src/parquet/level_comparison_simd_256.cc index 024353dbc855..4fbddf5a080c 100644 --- a/cpp/src/parquet/level_comparison_simd_256.cc +++ b/cpp/src/parquet/level_comparison_simd_256.cc @@ -17,6 +17,7 @@ #if defined(ARROW_HAVE_RUNTIME_AVX2) # define FIND_MIN_MAX_PLATFORM FindMinMaxAvx2 +# define GREATER_THAN_BITMAP_PLATFORM GreaterThanBitmapAvx2 # define ARCH_PLATFORM xsimd::avx2 #endif @@ -33,7 +34,13 @@ MinMax FIND_MIN_MAX_PLATFORM(const int16_t* levels, int64_t num_levels) { return FindMinMaxSimd(levels, num_levels); } +uint64_t GREATER_THAN_BITMAP_PLATFORM(const int16_t* levels, int64_t num_levels, + int16_t rhs) { + return GreaterThanBitmapSimd(levels, num_levels, rhs); +} + } // namespace parquet::internal #undef ARCH_PLATFORM +#undef GREATER_THAN_BITMAP_PLATFORM #undef FIND_MIN_MAX_PLATFORM diff --git a/cpp/src/parquet/level_comparison_simd_internal.h b/cpp/src/parquet/level_comparison_simd_internal.h index 518ce16f39e8..205b3f83003c 100644 --- a/cpp/src/parquet/level_comparison_simd_internal.h +++ b/cpp/src/parquet/level_comparison_simd_internal.h @@ -25,14 +25,17 @@ namespace parquet::internal { #if defined(ARROW_HAVE_NEON) MinMax FindMinMaxNeon(const int16_t* levels, int64_t num_levels); +uint64_t GreaterThanBitmapNeon(const int16_t* levels, int64_t num_levels, int16_t rhs); #endif #if defined(ARROW_HAVE_SSE4_2) || defined(ARROW_HAVE_RUNTIME_SSE4_2) MinMax FindMinMaxSse42(const int16_t* levels, int64_t num_levels); +uint64_t GreaterThanBitmapSse42(const int16_t* levels, int64_t num_levels, int16_t rhs); #endif #if defined(ARROW_HAVE_RUNTIME_AVX2) MinMax FindMinMaxAvx2(const int16_t* levels, int64_t num_levels); +uint64_t GreaterThanBitmapAvx2(const int16_t* levels, int64_t num_levels, int16_t rhs); #endif } // namespace parquet::internal diff --git a/cpp/src/parquet/level_comparison_simd_kernel_internal.h b/cpp/src/parquet/level_comparison_simd_kernel_internal.h index 5b15a8f77b01..eeeffd81188d 100644 --- a/cpp/src/parquet/level_comparison_simd_kernel_internal.h +++ b/cpp/src/parquet/level_comparison_simd_kernel_internal.h @@ -23,6 +23,7 @@ #include #include +#include "arrow/util/endian.h" #include "parquet/level_comparison.h" namespace parquet::internal { @@ -53,4 +54,26 @@ MinMax FindMinMaxSimd(const int16_t* levels, int64_t num_levels) { return out; } +/// Build a bitmap (LSB first, little-endian) with bit i set iff levels[i] > rhs. +/// +/// \param[in] num_levels must be in [0, 64]. +template +uint64_t GreaterThanBitmapSimd(const int16_t* levels, int64_t num_levels, int16_t rhs) { + using batch = xsimd::batch; + constexpr int64_t kLanes = static_cast(batch::size); + static_assert(kLanes <= 64); + + const batch vrhs(rhs); + uint64_t mask = 0; + int64_t i = 0; + for (; i + kLanes <= num_levels; i += kLanes) { + const auto gt = batch::load_unaligned(levels + i) > vrhs; + mask |= static_cast(gt.mask()) << i; + } + for (; i < num_levels; ++i) { + mask |= static_cast(levels[i] > rhs) << i; + } + return ::arrow::bit_util::ToLittleEndian(mask); +} + } // namespace parquet::internal