From 61ca7a99796b31c96dfd3a934d1d0de7e812d7f8 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Wed, 1 Jul 2026 21:57:07 +0100 Subject: [PATCH 01/15] GH-50295: [C++][R] #include in vector_select_k.cc breaks macOS CRAN and wasm builds (#50297) ### Rationale for this change R build failures due CRAN toolchain ### What changes are included in this PR? Use version of functions available on CRAN toolchain ### Are these changes tested? By existing CI jobs, and additional unit tests. ### Are there any user-facing changes? No * GitHub Issue: #50295 Lead-authored-by: Nic Crane Co-authored-by: Antoine Pitrou Signed-off-by: Nic Crane --- .../arrow/compute/kernels/select_k_test.cc | 366 ++++++++++++++++-- .../arrow/compute/kernels/vector_select_k.cc | 18 +- 2 files changed, 331 insertions(+), 53 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/select_k_test.cc b/cpp/src/arrow/compute/kernels/select_k_test.cc index 67e5d214636b..47e4af58001c 100644 --- a/cpp/src/arrow/compute/kernels/select_k_test.cc +++ b/cpp/src/arrow/compute/kernels/select_k_test.cc @@ -276,6 +276,26 @@ TEST_F(TestSelectKWithArray, FullSelectKNull) { Check(uint8(), array_input, options, expected); } +TEST_F(TestSelectKWithArray, PartialSelectKAllNull) { + auto array_input = R"([null, null, null, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(4, sort_keys); + auto expected = R"([null, null, null, null])"; + Check(uint8(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + Check(uint8(), array_input, options, expected); +} + +TEST_F(TestSelectKWithArray, FullSelectKAllNull) { + auto array_input = R"([null, null, null, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = R"([null, null, null, null, null])"; + Check(uint8(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + Check(uint8(), array_input, options, expected); +} + TEST_F(TestSelectKWithArray, PartialSelectKNullNaN) { auto array_input = R"([null, 30, NaN, 20, 10, null])"; std::vector sort_keys{SortKey("a", SortOrder::Descending)}; @@ -293,9 +313,31 @@ TEST_F(TestSelectKWithArray, FullSelectKNullNaN) { options.sort_keys[0].null_placement = NullPlacement::AtStart; Check(float64(), array_input, options, "[null, null, NaN, 30, 20, 10]"); } + +TEST_F(TestSelectKWithArray, PartialSelectKAllNullAndNaN) { + auto array_input = R"([null, NaN, NaN, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(4, sort_keys); + auto expected = R"([NaN, NaN, null, null])"; + Check(float64(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = R"([null, null, null, NaN])"; + Check(float64(), array_input, options, expected); +} + +TEST_F(TestSelectKWithArray, FullSelectKAllNullAndNaN) { + auto array_input = R"([null, NaN, NaN, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = R"([NaN, NaN, null, null, null])"; + Check(float64(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = R"([null, null, null, NaN, NaN])"; + Check(float64(), array_input, options, expected); +} + // Test basic cases for chunked array -template struct TestSelectKWithChunkedArray : public ::testing::Test { TestSelectKWithChunkedArray() {} @@ -323,6 +365,15 @@ struct TestSelectKWithChunkedArray : public ::testing::Test { AssertSelectK(chunked_array, k); } + void Check(const std::shared_ptr& type, const std::vector& input, + const SelectKOptions& options, const std::string& expected) { + std::shared_ptr actual; + auto input_array = ChunkedArrayFromJSON(type, input); + auto expected_array = ChunkedArrayFromJSON(type, {expected}); + ASSERT_OK(this->DoSelectK(input_array, options, &actual)); + AssertChunkedEqual(*expected_array, *actual); + } + void Check(const std::shared_ptr& chunked_array, const SelectKOptions& options, const std::shared_ptr& expected_array) { @@ -343,9 +394,12 @@ struct TestSelectKWithChunkedArray : public ::testing::Test { } }; -TYPED_TEST_SUITE(TestSelectKWithChunkedArray, SelectKableTypes); +template +struct TestSelectKWithChunkedArrayTyped : public TestSelectKWithChunkedArray {}; + +TYPED_TEST_SUITE(TestSelectKWithChunkedArrayTyped, SelectKableTypes); -TYPED_TEST(TestSelectKWithChunkedArray, RandomValuesWithSlices) { +TYPED_TEST(TestSelectKWithChunkedArrayTyped, RandomValuesWithSlices) { Random rand(0x61549225); int length = 100; for (auto null_probability : {0.0, 0.1, 0.5, 1.0}) { @@ -361,62 +415,119 @@ TYPED_TEST(TestSelectKWithChunkedArray, RandomValuesWithSlices) { } } -TYPED_TEST(TestSelectKWithChunkedArray, PartialSelectKNull) { - auto chunked_array = ChunkedArrayFromJSON(uint8(), { - "[null, 1]", - "[3, null, 2]", - "[1]", - }); +TEST_F(TestSelectKWithChunkedArray, PartialSelectKNull) { + auto chunked_array = std::vector{ + "[null, 1]", + "[3, null, 2]", + "[1]", + }; std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; auto options = SelectKOptions(3, sort_keys); - auto expected = ChunkedArrayFromJSON(uint8(), {"[1, 1, 2]"}); - this->Check(chunked_array, options, expected); + auto expected = "[1, 1, 2]"; + this->Check(uint8(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtStart; - expected = ChunkedArrayFromJSON(uint8(), {"[null, null, 1]"}); - this->Check(chunked_array, options, expected); + expected = "[null, null, 1]"; + this->Check(uint8(), chunked_array, options, expected); } -TYPED_TEST(TestSelectKWithChunkedArray, FullSelectKNull) { - auto chunked_array = ChunkedArrayFromJSON(uint8(), { - "[null, 1]", - "[3, null, 2]", - "[1]", - }); +TEST_F(TestSelectKWithChunkedArray, FullSelectKNull) { + auto chunked_array = std::vector{ + "[null, 1]", + "[3, null, 2]", + "[1]", + }; std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; auto options = SelectKOptions(10, sort_keys); options.sort_keys[0].null_placement = NullPlacement::AtStart; - auto expected = ChunkedArrayFromJSON(uint8(), {"[null, null, 1, 1, 2, 3]"}); - this->Check(chunked_array, options, expected); + auto expected = "[null, null, 1, 1, 2, 3]"; + this->Check(uint8(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtEnd; - expected = ChunkedArrayFromJSON(uint8(), {"[1, 1, 2, 3, null, null]"}); - this->Check(chunked_array, options, expected); + expected = "[1, 1, 2, 3, null, null]"; + this->Check(uint8(), chunked_array, options, expected); } -TYPED_TEST(TestSelectKWithChunkedArray, PartialSelectKNullNaN) { - auto chunked_array = ChunkedArrayFromJSON( - float64(), {"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}); +TEST_F(TestSelectKWithChunkedArray, PartialSelectKAllNull) { + auto chunked_array = std::vector{ + "[null, null]", + "[null, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(3, sort_keys); + auto expected = "[null, null, null]"; + this->Check(uint8(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + this->Check(uint8(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, FullSelectKAllNull) { + auto chunked_array = std::vector{ + "[null, null]", + "[null, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = "[null, null, null, null, null, null]"; + this->Check(uint8(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + this->Check(uint8(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, PartialSelectKNullNaN) { + auto chunked_array = + std::vector{"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}; std::vector sort_keys{SortKey("a", SortOrder::Descending)}; auto options = SelectKOptions(3, sort_keys); options.sort_keys[0].null_placement = NullPlacement::AtStart; - auto expected = ChunkedArrayFromJSON(float64(), {"[null, null, NaN]"}); - this->Check(chunked_array, options, expected); + auto expected = "[null, null, NaN]"; + this->Check(float64(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtEnd; - expected = ChunkedArrayFromJSON(float64(), {"[10, 3, 2]"}); - this->Check(chunked_array, options, expected); + expected = "[10, 3, 2]"; + this->Check(float64(), chunked_array, options, expected); } -TYPED_TEST(TestSelectKWithChunkedArray, FullSelectKNullNaN) { - auto chunked_array = ChunkedArrayFromJSON( - float64(), {"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}); +TEST_F(TestSelectKWithChunkedArray, FullSelectKNullNaN) { + auto chunked_array = + std::vector{"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}; std::vector sort_keys{SortKey("a", SortOrder::Descending)}; auto options = SelectKOptions(10, sort_keys); options.sort_keys[0].null_placement = NullPlacement::AtStart; - auto expected = - ChunkedArrayFromJSON(float64(), {"[null, null, NaN, NaN, 10, 3, 2, 1, 1]"}); - this->Check(chunked_array, options, expected); + auto expected = "[null, null, NaN, NaN, 10, 3, 2, 1, 1]"; + this->Check(float64(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtEnd; - expected = ChunkedArrayFromJSON(float64(), {"[10, 3, 2, 1, 1, NaN, NaN, null, null]"}); - this->Check(chunked_array, options, expected); + expected = "[10, 3, 2, 1, 1, NaN, NaN, null, null]"; + this->Check(float64(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, PartialSelectKAllNullAndNaN) { + auto chunked_array = std::vector{ + "[null, NaN]", + "[NaN, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(3, sort_keys); + auto expected = "[NaN, NaN, null]"; + this->Check(float64(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = "[null, null, null]"; + this->Check(float64(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, FullSelectKAllNullAndNaN) { + auto chunked_array = std::vector{ + "[null, NaN]", + "[NaN, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = "[NaN, NaN, null, null, null, null]"; + this->Check(float64(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = "[null, null, null, null, NaN, NaN]"; + this->Check(float64(), chunked_array, options, expected); } template @@ -790,6 +901,39 @@ TEST_F(TestSelectKWithRecordBatch, PartialSelectKNullNaN) { Check(schema, batch_input, options, expected); } +TEST_F(TestSelectKWithRecordBatch, PartialSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + auto batch_input = R"([ + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN}, + {"a": NaN, "b": NaN}, + {"a": null, "b": NaN} + ])"; + std::vector sort_keys{ + SortKey("a", SortOrder::Ascending, NullPlacement::AtStart), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(3, sort_keys); + auto expected = R"([{"a": null, "b": NaN}, + {"a": null, "b": NaN}, + {"a": null, "b": null}])"; + Check(schema, batch_input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}])"; + Check(schema, batch_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtEnd; + expected = R"([{"a": NaN, "b": null}, + {"a": NaN, "b": NaN}, + {"a": null, "b": null}])"; + Check(schema, batch_input, options, expected); +} + TEST_F(TestSelectKWithRecordBatch, FullSelectKNullNaN) { auto schema = ::arrow::schema({ {field("a", float32())}, @@ -834,6 +978,51 @@ TEST_F(TestSelectKWithRecordBatch, FullSelectKNullNaN) { Check(schema, batch_input, options, expected); } +TEST_F(TestSelectKWithRecordBatch, FullSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + auto batch_input = R"([ + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN}, + {"a": NaN, "b": NaN}, + {"a": null, "b": NaN} + ])"; + std::vector sort_keys{ + SortKey("a", SortOrder::Ascending, NullPlacement::AtStart), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = R"([{"a": null, "b": NaN}, + {"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": NaN}, + {"a": NaN, "b": null} + ])"; + Check(schema, batch_input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}, + {"a": null, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"; + Check(schema, batch_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtEnd; + expected = R"([{"a": NaN, "b": null}, + {"a": NaN, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}, + {"a": null, "b": NaN} + ])"; + Check(schema, batch_input, options, expected); +} + TEST_F(TestSelectKWithRecordBatch, BottomKOneColumnKey) { auto schema = ::arrow::schema({ {field("country", utf8())}, @@ -1147,26 +1336,26 @@ TEST_F(TestSelectKWithTable, FullSelectKNullNaN) { {"a": 1, "b": 3}, {"a": 3, "b": null}, {"a": 6, "b": NaN}, - {"a": 6, "b": null}, + {"a": 6, "b": null}, {"a": NaN, "b": 5}, - {"a": null, "b": 5}, + {"a": null, "b": 5}, {"a": null, "b": null}])"}; Check(schema, input, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtStart; expected = {R"([ - {"a": null, "b": 5}, + {"a": null, "b": 5}, {"a": null, "b": null}, {"a": NaN, "b": 5}, {"a": 1, "b": 5}, {"a": 1, "b": 3}, {"a": 3, "b": null}, {"a": 6, "b": NaN}, - {"a": 6, "b": null} + {"a": 6, "b": null} ])"}; Check(schema, input, options, expected); options.sort_keys[1].null_placement = NullPlacement::AtStart; expected = {R"([ - {"a": null, "b": null}, + {"a": null, "b": null}, {"a": null, "b": 5}, {"a": NaN, "b": 5}, {"a": 1, "b": 5}, @@ -1178,5 +1367,96 @@ TEST_F(TestSelectKWithTable, FullSelectKNullNaN) { Check(schema, input, options, expected); } +TEST_F(TestSelectKWithTable, PartialSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + std::vector input = { + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN} + ])", + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"}; + + std::vector sort_keys{SortKey("a", SortOrder::Ascending), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(3, sort_keys); + + std::vector expected = { + R"([{"a": NaN, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN} + ])"}; + Check(schema, input, options, expected); +} + +TEST_F(TestSelectKWithTable, FullSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + std::vector input = { + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN} + ])", + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"}; + + std::vector sort_keys{SortKey("a", SortOrder::Ascending), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(10, sort_keys); + + std::vector expected = { + R"([{"a": NaN, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"}; + Check(schema, input, options, expected); +} + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/vector_select_k.cc b/cpp/src/arrow/compute/kernels/vector_select_k.cc index cc375919658c..ea072d179eee 100644 --- a/cpp/src/arrow/compute/kernels/vector_select_k.cc +++ b/cpp/src/arrow/compute/kernels/vector_select_k.cc @@ -17,7 +17,6 @@ #include #include -#include #include #include "arrow/compute/function.h" @@ -118,22 +117,22 @@ void HeapSortNonNullsToOutput(std::span non_null_input_range, Comparat return; } std::span heap = non_null_input_range.subspan(0, output_range.size()); - std::ranges::make_heap(heap, cmp); + std::make_heap(heap.begin(), heap.end(), cmp); std::span remaining_input = non_null_input_range.subspan(output_range.size()); for (uint64_t x_index : remaining_input) { if (cmp(x_index, heap.front())) { - std::ranges::pop_heap(heap, cmp); + std::pop_heap(heap.begin(), heap.end(), cmp); heap.back() = x_index; - std::ranges::push_heap(heap, cmp); + std::push_heap(heap.begin(), heap.end(), cmp); } } // fill output in reverse when destructing, // as the "worst" (next-to-would-have-been-replaced) element is at heap-top - for (auto& reverse_out_iter : std::ranges::reverse_view(output_range)) { - reverse_out_iter = heap.front(); // heap-top has the next element - std::ranges::pop_heap(heap, cmp); + for (int64_t i = output_range.size(); i > 0; --i) { + output_range[i - 1] = heap.front(); // heap-top has the next element + std::pop_heap(heap.begin(), heap.end(), cmp); // Decrease heap-size by one heap = heap.first(heap.size() - 1); } @@ -422,9 +421,8 @@ class ChunkedArraySelector : public TypeVisitor { // so the heap must have been completely filled DCHECK_EQ(heap.size(), output.non_null_like_range.size()); - for (uint64_t& reverse_out_iter : - std::ranges::reverse_view(output.non_null_like_range)) { - reverse_out_iter = + for (int64_t i = output.non_null_like_range.size(); i > 0; --i) { + output.non_null_like_range[i - 1] = heap.top().index + heap.top().offset; // heap-top has the next element heap.pop(); } From 35e0f631fa8363f243771177b3ef9d6886e897ed Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 2 Jul 2026 17:24:37 +0900 Subject: [PATCH 02/15] GH-50318: [R][CI] Install missing libpng-dev for test-r-linux-as-cran (#50328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change We need `libpng-dev` for the png R package. ### What changes are included in this PR? Install `libpng-dev`. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #50318 Authored-by: Sutou Kouhei Signed-off-by: Raúl Cumplido --- ci/scripts/r_install_system_dependencies.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/scripts/r_install_system_dependencies.sh b/ci/scripts/r_install_system_dependencies.sh index 237e0e9408b3..042d6a5c9adb 100755 --- a/ci/scripts/r_install_system_dependencies.sh +++ b/ci/scripts/r_install_system_dependencies.sh @@ -38,10 +38,11 @@ fi # Install curl, OpenSSL, and libuv # - curl/OpenSSL: technically only needed for S3/GCS support, but # installing the R curl package fails without it +# - libpng: required by the png R package # - libuv: required by the fs R package (no longer bundles libuv by default) case "$PACKAGE_MANAGER" in apt-get) - apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev + apt-get install -y libcurl4-openssl-dev libpng-dev libssl-dev libuv1-dev ;; apk) $PACKAGE_MANAGER add curl-dev openssl-dev libuv-dev From f83c8ca541aec922ea590e712a0616dc7f931a95 Mon Sep 17 00:00:00 2001 From: Antoine Prouvost Date: Thu, 2 Jul 2026 11:52:47 +0200 Subject: [PATCH 03/15] GH-50330: [C++][R][Parquet] Add missing typename in RleBitPackedDecoderGetRunDecode (#50332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Fix compiler error. ### What changes are included in this PR? Missing typename keyword. ### Are these changes tested? In CI. ### Are there any user-facing changes? No. * GitHub Issue: #50330 Authored-by: AntoinePrv Signed-off-by: Raúl Cumplido --- cpp/src/arrow/util/rle_encoding_internal.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/util/rle_encoding_internal.h b/cpp/src/arrow/util/rle_encoding_internal.h index 825cd253df9b..5dc94f368d3c 100644 --- a/cpp/src/arrow/util/rle_encoding_internal.h +++ b/cpp/src/arrow/util/rle_encoding_internal.h @@ -811,7 +811,8 @@ auto RleBitPackedDecoder::GetBatch(value_type* out, } parser_.ParseWithCallable([&](auto run) { - using RunDecoder = RleBitPackedDecoderGetRunDecoder::type; + using RunDecoder = + typename RleBitPackedDecoderGetRunDecoder::type; ARROW_DCHECK_LT(values_read, batch_size); RunDecoder decoder(run, value_bit_width_); @@ -1124,7 +1125,8 @@ auto RleBitPackedDecoder::GetSpaced(Converter converter, } parser_.ParseWithCallable([&](auto run) { - using RunDecoder = RleBitPackedDecoderGetRunDecoder::type; + using RunDecoder = + typename RleBitPackedDecoderGetRunDecoder::type; RunDecoder decoder(run, value_bit_width_); @@ -1306,7 +1308,8 @@ auto RleBitPackedDecoder::GetBatchWithDict(const V* dictionary, } parser_.ParseWithCallable([&](auto run) { - using RunDecoder = RleBitPackedDecoderGetRunDecoder::type; + using RunDecoder = + typename RleBitPackedDecoderGetRunDecoder::type; RunDecoder decoder(run, value_bit_width_); From c658548ff58d48a378f83998eccd0a8c30adf0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 2 Jul 2026 23:41:04 +0200 Subject: [PATCH 04/15] GH-50293: [CI] Run check-labels for all triggers to avoid cancelling further steps and add tag to set_enabled (#50340) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Currently when check-labels is not a pull_request even the following steps are cancelled. Also we don't enable tags execution which is required for releases. ### What changes are included in this PR? Run check-labels for the specified events and enable jobs for tags. ### Are these changes tested? I've tested on my fork by pushing to main and creating tags, more details on the comment on the PR. ### Are there any user-facing changes? No * GitHub Issue: #50293 Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- .github/workflows/cpp_extra.yml | 2 +- .github/workflows/cuda_extra.yml | 2 +- .github/workflows/package_linux.yml | 2 +- .github/workflows/r_extra.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 9a863d2a5ff7..8b072ed90afa 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -66,7 +66,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: cpp_extra @@ -80,6 +79,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') diff --git a/.github/workflows/cuda_extra.yml b/.github/workflows/cuda_extra.yml index 64f2597fe89f..f58ee86e6fac 100644 --- a/.github/workflows/cuda_extra.yml +++ b/.github/workflows/cuda_extra.yml @@ -49,7 +49,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: cuda_extra @@ -63,6 +62,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: CUDA') diff --git a/.github/workflows/package_linux.yml b/.github/workflows/package_linux.yml index b5afb7f36a0d..b96627f49ff4 100644 --- a/.github/workflows/package_linux.yml +++ b/.github/workflows/package_linux.yml @@ -58,7 +58,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: package_linux @@ -72,6 +71,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: Package: Linux') diff --git a/.github/workflows/r_extra.yml b/.github/workflows/r_extra.yml index 1ba7d2247cef..142464dabc75 100644 --- a/.github/workflows/r_extra.yml +++ b/.github/workflows/r_extra.yml @@ -61,7 +61,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: r_extra @@ -75,6 +74,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: R') From fe2f85c7521da2e62320cc25d7e55a50304f4893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 2 Jul 2026 23:48:43 +0200 Subject: [PATCH 05/15] GH-50336: [Release][Archery] Fix archery GitHub integration for release scripts (#50337) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change The source verification and binary verification scripts comment to the Verification PR were failing due to the last updates to archery in order to use a single dependency. ### What changes are included in this PR? Add required base and head branches to be used when looking for a PR. Use create_issue_comment on PRs instead of create_comment which tries to create a review comment which requires more arguments. ### Are these changes tested? Yes, I've tested them locally in isolation in order to add comments on the verification PR. ### Are there any user-facing changes? No * GitHub Issue: #50336 Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- dev/archery/archery/crossbow/cli.py | 14 ++++++++++---- dev/release/03-binary-submit.sh | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index 35e0d4d2c022..56c319fe7ef0 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -208,7 +208,7 @@ def verify_release_candidate(obj, base_branch, create_pr, for flag, group in zip(verify_flags, verify_groups): if flag: job_groups += f" --group {group}" - response.create_comment( + response.create_issue_comment( f"{command} {job_groups} --param " + f"release={version} --param rc={rc}") @@ -300,6 +300,10 @@ def asset_callback(task_name, task, asset): @crossbow.command() +@click.option('--base-branch', default='main', + help='Set base branch for the PR.') +@click.option('--head-branch', default=None, + help='Set head branch for the PR.') @click.option('--arrow-remote', '-r', default=None, help='Set GitHub remote explicitly, which is going to be cloned ' 'on the CI services. Note, that no validation happens ' @@ -313,7 +317,8 @@ def asset_callback(task_name, task, asset): @click.option('--pr-title', required=True, help='Track the job submitted on PR with given title') @click.pass_obj -def report_pr(obj, arrow_remote, crossbow, fetch, job_name, pr_title): +def report_pr(obj, base_branch, head_branch, arrow_remote, crossbow, + fetch, job_name, pr_title): arrow = obj['arrow'] queue = obj['queue'] if fetch: @@ -322,11 +327,12 @@ def report_pr(obj, arrow_remote, crossbow, fetch, job_name, pr_title): report = CommentReport(job, crossbow_repo=crossbow) target_arrow = Repo(path=arrow.path, remote_url=arrow_remote) - pull_request = target_arrow.github_pr(title=pr_title, + pull_request = target_arrow.github_pr(base=base_branch, head=head_branch, + title=pr_title, github_token=queue.github_token, create=False) # render the response comment's content on the PR - pull_request.create_comment(report.show()) + pull_request.create_issue_comment(report.show()) click.echo(f'Job is tracked on PR {pull_request.html_url}') diff --git a/dev/release/03-binary-submit.sh b/dev/release/03-binary-submit.sh index e3a0fc4ee7a1..acd0af5be49b 100755 --- a/dev/release/03-binary-submit.sh +++ b/dev/release/03-binary-submit.sh @@ -34,6 +34,7 @@ version_with_rc="${version}-rc${rc}" crossbow_job_prefix="release-${version_with_rc}" release_tag="apache-arrow-${version}-rc${rc}" rc_branch="release-${version_with_rc}" +maint_branch="maint-${version}" : ${ARROW_REPOSITORY:="apache/arrow"} : ${ARROW_BRANCH:=${release_tag}} @@ -57,5 +58,7 @@ job_name=$(archery crossbow latest-prefix --no-fetch ${crossbow_job_prefix}) archery crossbow report-pr \ --no-fetch \ --arrow-remote "https://github.com/${ARROW_REPOSITORY}" \ + --base-branch ${maint_branch} \ + --head-branch ${rc_branch} \ --job-name ${job_name} \ --pr-title "WIP: [Release] Verify ${rc_branch}" From 7f72d5bc814b2c5bbd3eec0c9c66379508ad2d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 3 Jul 2026 09:48:04 +0200 Subject: [PATCH 06/15] MINOR: [Release] Update CHANGELOG.md for 25.0.0 --- CHANGELOG.md | 231 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6101f5d3cac2..3c653e79c318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,235 @@ +# Apache Arrow 25.0.0 (2026-07-07 00:00:00+00:00) + +## Bug Fixes + +* [GH-17081](https://github.com/apache/arrow/issues/17081) - [C++] arrow::stl::TupleRangeFromTable docs incorrect (#50095) +* [GH-20403](https://github.com/apache/arrow/issues/20403) - [Doc] pyarrow.Array.diff Examples is wrongly rendered (#50096) +* [GH-32994](https://github.com/apache/arrow/issues/32994) - [Dev][Archery] Fix multi-arch Docker configuration (#50125) +* [GH-36503](https://github.com/apache/arrow/issues/36503) - [C++] Make DictionaryArray::dictionary() thread-safe (#48905) +* [GH-38558](https://github.com/apache/arrow/issues/38558) - [C++] Add support for null sort option per sort key (#46926) +* [GH-39603](https://github.com/apache/arrow/issues/39603) - [R] Error: Cannot convert Dictionary Array of type dictionary to R (#49710) +* [GH-39754](https://github.com/apache/arrow/issues/39754) - [R] Docs are not clear on expected behaviour of date parsing functions (e.g. dmy()) on Windows vs. Linux/MacOS (#49708) +* [GH-39784](https://github.com/apache/arrow/issues/39784) - [C++][Gandiva] Fix decimal in_expr crash with cached object code (#49951) +* [GH-40410](https://github.com/apache/arrow/issues/40410) - [C++] Skip only s3fs-tests and s3fs-module-tests that require MinIO if MinIO is not available (#50215) +* [GH-40640](https://github.com/apache/arrow/issues/40640) - [R] to_arrow() loses group_by() (#49713) +* [GH-40742](https://github.com/apache/arrow/issues/40742) - [R] fix max_rows_per_group must be a positive number (#49709) +* [GH-40886](https://github.com/apache/arrow/issues/40886) - [R] Cryptic error when creating Arrow array from POSIXct with invalid time zones (#49714) +* [GH-43574](https://github.com/apache/arrow/issues/43574) - [Python][Parquet] do not add partition columns from file path when reading single file (#49853) +* [GH-45193](https://github.com/apache/arrow/issues/45193) - [C++][Compute] Treat NaNs and nulls as distinct values in rank tie-breaking (#49304) +* [GH-46179](https://github.com/apache/arrow/issues/46179) - [Python] Bump index level once if pandas df already contains __index_level_i__ column (#46884) +* [GH-47100](https://github.com/apache/arrow/issues/47100) - [Docs] Correct the Statistics schema specification (#50092) +* [GH-47252](https://github.com/apache/arrow/issues/47252) - [C++][Compute] Fix sort_indices for temporal types in arrow::Table (#50270) +* [GH-47447](https://github.com/apache/arrow/issues/47447) - [C++] Fix replace_with_mask for null type arrays (#49950) +* [GH-47481](https://github.com/apache/arrow/issues/47481) - [C++][Acero] record_batch_reader_source specify Ordering::Implicit to support `select * limit k` (#47482) +* [GH-47642](https://github.com/apache/arrow/issues/47642) - [C++] Catch exceptions from initial_task in AsyncTaskScheduler (#49860) +* [GH-47657](https://github.com/apache/arrow/issues/47657) - [C++][Parquet] Check for integer overflow when coercing timestamps (#49615) +* [GH-48094](https://github.com/apache/arrow/issues/48094) - [C++] Restrict SecureString capacity tail check to Linux (#49906) +* [GH-48254](https://github.com/apache/arrow/issues/48254) - [Python][Parquet] Support extension types in read_schema (#48255) +* [GH-48712](https://github.com/apache/arrow/issues/48712) - [R] "Invalid metadata$r" warning (#49608) +* [GH-48801](https://github.com/apache/arrow/issues/48801) - [C++] Set CMAKE_POLICY_VERSION_MINIMUM for RapidJSON (#49993) +* [GH-48926](https://github.com/apache/arrow/issues/48926) - [C++] Upgrade Abseil/Protobuf/GRPC/Google-Cloud-CPP bundled versions (#48964) +* [GH-49272](https://github.com/apache/arrow/issues/49272) - [C++][CI] Fix intermittent segfault in arrow-json-test with MinGW (#49462) +* [GH-49327](https://github.com/apache/arrow/issues/49327) - [Python][Packaging] Use ARROW_SIMD_LEVEL=NEON for macOS arm64 (#50181) +* [GH-49433](https://github.com/apache/arrow/issues/49433) - [C++] Buffer ARROW_LOG output to prevent thread interleaving (#49663) +* [GH-49465](https://github.com/apache/arrow/issues/49465) - [CI][C++] Fix Abseil hang in arrow-flight-test on ODBC Windows (#50085) +* [GH-49522](https://github.com/apache/arrow/issues/49522) - [CI] Update chrome_version for emscripten job to latest stable (v148) (#49523) +* [GH-49614](https://github.com/apache/arrow/issues/49614) - [C++] Report an error instead of silent truncation in base64_decode on invalid input (#49660) +* [GH-49689](https://github.com/apache/arrow/issues/49689) - [R][C++] Parquets do not support list-columns of ordered factors (ordered dictionaries) (#49937) +* [GH-49719](https://github.com/apache/arrow/issues/49719) - [C++] Rename vendored date header guards (#49778) +* [GH-49740](https://github.com/apache/arrow/issues/49740) - [C++][Python] Fix casts to view types leaving null variadic buffers (#50166) +* [GH-49743](https://github.com/apache/arrow/issues/49743) - [Release] Split the vote thread preparation into its own shell script (#49770) +* [GH-49745](https://github.com/apache/arrow/issues/49745) - [Docs][Python] Fix doctests failure in substrait.rst (#49754) +* [GH-49752](https://github.com/apache/arrow/issues/49752) - [C++][Gandiva] Fix potential buffer overrun in Gandiva SSL function (#49780) +* [GH-49753](https://github.com/apache/arrow/issues/49753) - [C++][Gandiva] Fix overflow in string functions (#49813) +* [GH-49757](https://github.com/apache/arrow/issues/49757) - [Release][CI] export SSL_CERT_FILE to bypass incompatibility with OpenSSL on RHEL-8 (#49769) +* [GH-49759](https://github.com/apache/arrow/issues/49759) - [C++][Integration] Harden BinaryView JSON parsing with runtime validation (#49758) +* [GH-49764](https://github.com/apache/arrow/issues/49764) - [C++][Python] Avoid building bundled Abseil outside resolve_dependency (#49936) +* [GH-49767](https://github.com/apache/arrow/issues/49767) - [CI][C++] Disable mold on Ubuntu 24.04 to work around mold#1247 (#50033) +* [GH-49803](https://github.com/apache/arrow/issues/49803) - [C++][CI] Avoid aborting when fuzz-mutated IPC file has less batches (#49804) +* [GH-49823](https://github.com/apache/arrow/issues/49823) - [CI] Add missing contents: read permission to Package Linux workflow (#49825) +* [GH-49831](https://github.com/apache/arrow/issues/49831) - [Python] Withhold annotations from Python wheel until they are complete (#50168) +* [GH-49834](https://github.com/apache/arrow/issues/49834) - [C++] Avoid building re2 unit tests (#50152) +* [GH-49837](https://github.com/apache/arrow/issues/49837) - [C++][Parquet] Avoid unbounded temporary std::vector in DELTA_(LENGTH_)BYTE_ARRAY decoder (#49838) +* [GH-49846](https://github.com/apache/arrow/issues/49846) - [CI][Python] fix test-conda-python-3.11-hypothesis (#49847) +* [GH-49861](https://github.com/apache/arrow/issues/49861) - [R] Missing R libarrow binary for linux-arm64 (#49893) +* [GH-49866](https://github.com/apache/arrow/issues/49866) - [R][Release] Restore using tzdb on Windows for tzdata (#49867) +* [GH-49872](https://github.com/apache/arrow/issues/49872) - [C++] Remove deprecated std::is_trivial (#49871) +* [GH-49875](https://github.com/apache/arrow/issues/49875) - [Python] Fix timezone dropped when converting tz-aware Categorical to Arrow array (#49878) +* [GH-49888](https://github.com/apache/arrow/issues/49888) - [C++][Compute] Fix count for run-end encoded arrays with nulls (#49908) +* [GH-49896](https://github.com/apache/arrow/issues/49896) - [C++] Reject short buffer reads in IPC reader (#49897) +* [GH-49905](https://github.com/apache/arrow/issues/49905) - [Archery] Fix archery benchmark diff with pandas 3 (#49912) +* [GH-49917](https://github.com/apache/arrow/issues/49917) - [Python] Remove Py_XDECREF to avoid Use-After-Free on `PyList_SetItem` in `SparseCSFTensorToNdarray` (#49916) +* [GH-49923](https://github.com/apache/arrow/issues/49923) - [Parquet][Python] Inconsistent default values for Parquet pre_buffer (#49924) +* [GH-49927](https://github.com/apache/arrow/issues/49927) - [Python][Parquet] Expose bloom_filter_offset and bloom_filter_length to Python in column chunk metadata (#49926) +* [GH-49930](https://github.com/apache/arrow/issues/49930) - [CI][C++] Pin MinGW MSYS2 packages to unblock CI (#49931) +* [GH-49933](https://github.com/apache/arrow/issues/49933) - [Python] Fix test_table_column_subset_metadata to set freq on correct object (#49944) +* [GH-49942](https://github.com/apache/arrow/issues/49942) - [Python] Protect PyBuffer and NumPyBuffer destructors against interpreter finalization (#49943) +* [GH-49948](https://github.com/apache/arrow/issues/49948) - [CI][C++] Revert PR 49931 (Pin MinGW MSYS2 packages) but keep bumped minIO version (#49945) +* [GH-49956](https://github.com/apache/arrow/issues/49956) - [GLib] Add fallback data type for unknown extension data type (#49969) +* [GH-49966](https://github.com/apache/arrow/issues/49966) - [C++] Detect different endianness between IPC file and stream in IPC file fuzzer (#49968) +* [GH-49974](https://github.com/apache/arrow/issues/49974) - [C++][CMake] Add missing result variable initialization for `validate_apple_libtool()` (#49975) +* [GH-49991](https://github.com/apache/arrow/issues/49991) - [C++][FlightRPC] Fix unity build ordering issue (#49990) +* [GH-49994](https://github.com/apache/arrow/issues/49994) - [CI] Remove `brew uninstall pkg-config` workarounds (#49997) +* [GH-49998](https://github.com/apache/arrow/issues/49998) - [CI][Python] Pin to an older release of miniforge to fix mamba hang (#49999) +* [GH-50000](https://github.com/apache/arrow/issues/50000) - [C++][FlightRPC] Use grpcpp/grpcpp.h not grpcpp/version_info.h for old gRPC (#50001) +* [GH-50009](https://github.com/apache/arrow/issues/50009) - [R] FinalizeS3 segfaults for stale connection (#50081) +* [GH-50010](https://github.com/apache/arrow/issues/50010) - [C++][Parquet] Fix undefined behavior in TypedColumnWriterImpl::UpdateLevelHistogram (#50011) +* [GH-50012](https://github.com/apache/arrow/issues/50012) - [Python] Fix list_ storage crashes when values exceed int32 offsets (#50016) +* [GH-50017](https://github.com/apache/arrow/issues/50017) - [CI][Release] Remove deprecated -f (force) flag from conda create on Windows verification job (#50018) +* [GH-50037](https://github.com/apache/arrow/issues/50037) - [Python] test_table_uses_memory_pool flaky on macOS 14 job (#50045) +* [GH-50041](https://github.com/apache/arrow/issues/50041) - [CI][Python] Make test_string_to_tzinfo_pytz_fallback more robust for platforms supporting lower case tz names (#50042) +* [GH-50043](https://github.com/apache/arrow/issues/50043) - [C++][Python] Fix hash_any/hash_all on sliced boolean arrays (#50094) +* [GH-50051](https://github.com/apache/arrow/issues/50051) - [C++][Parquet] Avoid size overflow in WKBBuffer::ReadCoords (#50036) +* [GH-50065](https://github.com/apache/arrow/issues/50065) - [Packaging][CI][C++] Drop unused libboost-system-dev (#50066) +* [GH-50090](https://github.com/apache/arrow/issues/50090) - [Packaging] Avoid building CUDA for Debian forky due to missing nvidia-cuda-toolkit (#50279) +* [GH-50098](https://github.com/apache/arrow/issues/50098) - [C++][Parquet] Enable filesystem support when building Parquet utilities (#50099) +* [GH-50103](https://github.com/apache/arrow/issues/50103) - [C++] Missing iosfwd include in cpp/src/arrow/util/string_util.h (#50101) +* [GH-50105](https://github.com/apache/arrow/issues/50105) - [C++][Python] Fix sliced sparse union null checks (#50108) +* [GH-50109](https://github.com/apache/arrow/issues/50109) - [C++][Gandiva] Fix incorrect error messages (#50110) +* [GH-50113](https://github.com/apache/arrow/issues/50113) - [C++][Python] Fix `count` for sliced union arrays (#50114) +* [GH-50129](https://github.com/apache/arrow/issues/50129) - [C++][Parquet] Enable `ARROW_JSON` automatically with `PARQUET_REQUIRE_ENCRYPTION` (#50130) +* [GH-50133](https://github.com/apache/arrow/issues/50133) - [CI] Make extra labels entirely manual on PRs (#50256) +* [GH-50149](https://github.com/apache/arrow/issues/50149) - [C++][Parquet] Avoid process abort when encoding fuzzer encounters OOM (#50150) +* [GH-50156](https://github.com/apache/arrow/issues/50156) - [C++][Parquet] Ignore min/max for unknown column order (#50157) +* [GH-50163](https://github.com/apache/arrow/issues/50163) - [R] Bug: Partial matching on $metadata$r causes errors with schema metadata keys starting with "r" (#50178) +* [GH-50171](https://github.com/apache/arrow/issues/50171) - [CI][R] Build libarrow is failing for Linux arm64 (#50179) +* [GH-50173](https://github.com/apache/arrow/issues/50173) - [C++][Gandiva] Install tzdata-legacy to fix failing TestTime.TestCastTimestampWithTZ from gandiva/precompiled/time_test.cc (#50211) +* [GH-50174](https://github.com/apache/arrow/issues/50174) - [CI][Python] Fix debug Python job (#50207) +* [GH-50176](https://github.com/apache/arrow/issues/50176) - [Python] Explicitly pass exc_type=ImportError to importorskip pyarrow.* (#50177) +* [GH-50204](https://github.com/apache/arrow/issues/50204) - [C++] Fix S3FileSystem::MakeUri dropping the bucket on Windows MinGW (#50205) +* [GH-50210](https://github.com/apache/arrow/issues/50210) - [C++][Gandiva] Replace precompiled std::string to fix _Unwind_Resume JIT failure on JNI builds (#50214) +* [GH-50212](https://github.com/apache/arrow/issues/50212) - [C++][Parquet] Add PrintTo to fix failure in parquet-internals-test on test-conda-cpp-valgrind (#50213) +* [GH-50218](https://github.com/apache/arrow/issues/50218) - [C++][FlightRPC] Add missing sudo to fix ODBC Linux job (#50307) +* [GH-50227](https://github.com/apache/arrow/issues/50227) - [Docs][Python][Parquet] Update parquet default version text in parquet.rst (#50228) +* [GH-50240](https://github.com/apache/arrow/issues/50240) - [C++] Make IPC message decoding stricter (#50235) +* [GH-50242](https://github.com/apache/arrow/issues/50242) - [CI][Release] Fix flaky dylib load failure in macOS verify-rc build (#50243) +* [GH-50253](https://github.com/apache/arrow/issues/50253) - [CI] Manually install aws-sdk-cpp with brew in order to avoid lock (#50254) +* [GH-50257](https://github.com/apache/arrow/issues/50257) - [CI][Docs] Fix doxygen failing due to double backticks (#50259) +* [GH-50277](https://github.com/apache/arrow/issues/50277) - [CI][Python] Avoid using generators for test parametrization on newer Pytest (#50278) +* [GH-50291](https://github.com/apache/arrow/issues/50291) - [Python][Packaging] Stop using nightly build dependencies for building free-threaded wheels (#50315) +* [GH-50293](https://github.com/apache/arrow/issues/50293) - [CI] Run check-labels for all triggers to avoid cancelling further steps and add tag to set_enabled (#50340) +* [GH-50294](https://github.com/apache/arrow/issues/50294) - [C++][R] Missing typename in ulp_distance.cc breaks clang-15 builds (#50296) +* [GH-50295](https://github.com/apache/arrow/issues/50295) - [C++][R] #include in vector_select_k.cc breaks macOS CRAN and wasm builds (#50297) +* [GH-50300](https://github.com/apache/arrow/issues/50300) - [C++][CI] Update include to generated/Message_generated.h so Meson is able to find (#50301) +* [GH-50318](https://github.com/apache/arrow/issues/50318) - [R][CI] Install missing libpng-dev for test-r-linux-as-cran (#50328) +* [GH-50330](https://github.com/apache/arrow/issues/50330) - [C++][R][Parquet] Add missing typename in RleBitPackedDecoderGetRunDecode (#50332) +* [GH-50336](https://github.com/apache/arrow/issues/50336) - [Release][Archery] Fix archery GitHub integration for release scripts (#50337) + + +## New Features and Improvements + +* [GH-14796](https://github.com/apache/arrow/issues/14796) - [Dev][COMPONENT] " to issue title automatically (#49892) +* [GH-19667](https://github.com/apache/arrow/issues/19667) - [C++][Gandiva] Use arrow::Result for RegexUtil::SqlLikePatternToPcre (#49879) +* [GH-22232](https://github.com/apache/arrow/issues/22232) - [C++][Python] Introduce optional default_column_type parameter (#47663) +* [GH-31318](https://github.com/apache/arrow/issues/31318) - [Python] Add fixed-offset timezones to Hypothesis test strategy (#49844) +* [GH-32381](https://github.com/apache/arrow/issues/32381) - [C++] Improve error handling for hash table merges (#49512) +* [GH-33241](https://github.com/apache/arrow/issues/33241) - [Archery] Replace github3 with pygithub (#48886) +* [GH-33390](https://github.com/apache/arrow/issues/33390) - [R] Field-level metadata (#49631) +* [GH-33420](https://github.com/apache/arrow/issues/33420) - [R] Improve error message when providing a mix of readr and Arrow options (#50048) +* [GH-38849](https://github.com/apache/arrow/issues/38849) - [C++][Parquet] Add support for list view and large list view (#50160) +* [GH-40062](https://github.com/apache/arrow/issues/40062) - [C++][Python] Conversion of Table to Arrow Tensor (#41870) +* [GH-45187](https://github.com/apache/arrow/issues/45187) - [Ruby] Ensure initializing all rb_memory_view_t members (#50234) +* [GH-45331](https://github.com/apache/arrow/issues/45331) - [C++] Use xsimd for CPU feature detection (#49940) +* [GH-45819](https://github.com/apache/arrow/issues/45819) - [C++] Add OptionalBitmapAnd utility (#49848) +* [GH-46178](https://github.com/apache/arrow/issues/46178) - [R] source_node alignment warning (#50120) +* [GH-46369](https://github.com/apache/arrow/issues/46369) - [C++] Add key-value pairs to the FileSystemFactory interface and to S3Options (#50044) +* [GH-46914](https://github.com/apache/arrow/issues/46914) - [C++][FlightSQL] Remove boost/algorithm/string.h dependency (#50241) +* [GH-46956](https://github.com/apache/arrow/issues/46956) - [Docs][CI] Enable version switcher in local and PR preview build (#46957) +* [GH-47435](https://github.com/apache/arrow/issues/47435) - [Python][Parquet] Add direct key encryption/decryption API (#49667) +* [GH-47769](https://github.com/apache/arrow/issues/47769) - [C++] SVE dynamic dispatch (#49756) +* [GH-47876](https://github.com/apache/arrow/issues/47876) - [C++][FlightRPC] ODBC: macOS `.PKG` installer for Intel and ARM (#49766) +* [GH-48028](https://github.com/apache/arrow/issues/48028) - [Python][Packaging] Update quay.io base manylinux and musllinux images for Python wheels and remove cp313t (#50082) +* [GH-48068](https://github.com/apache/arrow/issues/48068) - [C++][FlightRPC] Linux ODBC: Configure Dremio instance to allow remote testing (#49695) +* [GH-48294](https://github.com/apache/arrow/issues/48294) - [Ruby] Add RecordBatch#merge (#50175) +* [GH-48408](https://github.com/apache/arrow/issues/48408) - [C++] Enable ULP-based float comparison (#49290) +* [GH-49232](https://github.com/apache/arrow/issues/49232) - [Python] deprecate feather python (#49590) +* [GH-49275](https://github.com/apache/arrow/issues/49275) - [Doc] Update docs to specify disclosure of AI on mailing list messages (#49277) +* [GH-49497](https://github.com/apache/arrow/issues/49497) - [FlightRPC] Add is_update field to ActionCreatePreparedStatementResult (#49498) +* [GH-49534](https://github.com/apache/arrow/issues/49534) - [R] Implement dplyr recode_values(), replace_values(), and replace_when() (#49536) +* [GH-49537](https://github.com/apache/arrow/issues/49537) - [C++][FlightRPC] Windows CI to Support ODBC DLL & MSI Signing (#49603) +* [GH-49552](https://github.com/apache/arrow/issues/49552) - [C++][FlightRPC][ODBC] Enable ODBC test build on Linux (#49668) +* [GH-49644](https://github.com/apache/arrow/issues/49644) - [Python] Support converting list of multi-dimensional arrays to FixedShapeTensor (#50203) +* [GH-49651](https://github.com/apache/arrow/issues/49651) - [C++][FlightRPC] Fix ODBC Linux test segmentation fault (#49688) +* [GH-49675](https://github.com/apache/arrow/issues/49675) - [Docs] Make "stale issues" more visible to potential contributors and document stale issue policy (#49703) +* [GH-49684](https://github.com/apache/arrow/issues/49684) - [MATLAB] Introduce deprecation warnings for "legacy" Feather V1 functions `featherread` and `featherwrite` (#49705) +* [GH-49686](https://github.com/apache/arrow/issues/49686) - [C++][FlightRPC][ODBC][Release] Create signing script for Windows FlightSQL ODBC build (#49788) +* [GH-49700](https://github.com/apache/arrow/issues/49700) - [R][CI][Dev] Use air precommit hook (#49701) +* [GH-49720](https://github.com/apache/arrow/issues/49720) - [C++] Optimize base64_decode validation using lookup table (#49748) +* [GH-49723](https://github.com/apache/arrow/issues/49723) - [C++][FlightRPC][ODBC] Update ODBC Documentation (#49851) +* [GH-49725](https://github.com/apache/arrow/issues/49725) - [CI] Use environment variables instead of template expressions in workflow run blocks (#49733) +* [GH-49728](https://github.com/apache/arrow/issues/49728) - [CI] Set persist-credentials: false in checkout actions (#49734) +* [GH-49729](https://github.com/apache/arrow/issues/49729) - [CI] Scope workflow permissions and secret inheritance (#49773) +* [GH-49737](https://github.com/apache/arrow/issues/49737) - [R][CI] Disable Rhub GCC 12 and GCC 13 with LTO jobs - images no longer match CRAN (#49739) +* [GH-49738](https://github.com/apache/arrow/issues/49738) - [R][CI] Re-enable GCC+LTO job once rhub has a GCC 15 image (#49795) +* [GH-49751](https://github.com/apache/arrow/issues/49751) - [Python] Add raw fd support to pa.OSFile (#49750) +* [GH-49772](https://github.com/apache/arrow/issues/49772) - [C++] Bump bundled mimalloc version (#49801) +* [GH-49776](https://github.com/apache/arrow/issues/49776) - [CI][C++] Install libc6-dbg in apt-based Linux C++ images (#50034) +* [GH-49783](https://github.com/apache/arrow/issues/49783) - [C++][FlightRPC][ODBC] Reuse connections across test suite (#49784) +* [GH-49785](https://github.com/apache/arrow/issues/49785) - [C++][FlightRPC][ODBC] Get ODBC tests passing on Linux (#49786) +* [GH-49789](https://github.com/apache/arrow/issues/49789) - [C++] Use `CMAKE_INSTALL_DOCDIR` instead of static `share/doc/${PROJECT_NAME}` (#49790) +* [GH-49793](https://github.com/apache/arrow/issues/49793) - [R] Update NEWS.md for 24.0.0 (#49794) +* [GH-49805](https://github.com/apache/arrow/issues/49805) - [C++][Parquet] Avoid unbounded temporary allocation in DeltaBitPackDecoder::DecodeArrow (#49806) +* [GH-49807](https://github.com/apache/arrow/issues/49807) - [CI] Remove obsolete test-ubuntu-22.04-cpp-20 job (#49827) +* [GH-49835](https://github.com/apache/arrow/issues/49835) - [C++] A constexpr dynamic dispatch with static dispatch when possible (#49840) +* [GH-49890](https://github.com/apache/arrow/issues/49890) - [Dev] Group files under component comment headers in `.github/CODEOWNERS` (#49891) +* [GH-49898](https://github.com/apache/arrow/issues/49898) - [C++][CI] Use mold in more builds (#49899) +* [GH-49901](https://github.com/apache/arrow/issues/49901) - [R] Bump minimum supported R version to 4.2 now that 4.6 is out (#49929) +* [GH-49913](https://github.com/apache/arrow/issues/49913) - [Archery] Add preserve-dir and improve directory layout (#50056) +* [GH-49918](https://github.com/apache/arrow/issues/49918) - [C++][Parquet] Catch std::vector allocation errors in encoding fuzzer (#49919) +* [GH-49921](https://github.com/apache/arrow/issues/49921) - [C++] Bump xsimd to 14.2.0 (#49922) +* [GH-49938](https://github.com/apache/arrow/issues/49938) - [C++] Bump bundled c-ares to 1.34.6 (#49939) +* [GH-49946](https://github.com/apache/arrow/issues/49946) - [Format] Better document equivalence between IPC file and streams (#49947) +* [GH-49952](https://github.com/apache/arrow/issues/49952) - [C++][Gandiva] Use timegm in date_time_test utilities (#49953) +* [GH-49959](https://github.com/apache/arrow/issues/49959) - [C++][Parquet] Avoid unbounded temp alloc in BYTE_STREAM_SPLIT decoder (#49960) +* [GH-49967](https://github.com/apache/arrow/issues/49967) - [Python][CI] Raise oldest NumPy wheel-test requirement to a patched release (#49965) +* [GH-49981](https://github.com/apache/arrow/issues/49981) - [R][Packaging] Support building R package under r-universe/r-wasm (#49982) +* [GH-49988](https://github.com/apache/arrow/issues/49988) - [CI][Packaging] Enable reproducible builds on host for APT based Linux packages (#48148) +* [GH-50005](https://github.com/apache/arrow/issues/50005) - [C++] Use FetchContent for RapidJSON (#50006) +* [GH-50007](https://github.com/apache/arrow/issues/50007) - [C++][Parquet] Add bloom filter folding to automatically size SBBF filters (#50008) +* [GH-50014](https://github.com/apache/arrow/issues/50014) - [R] Replace imported symbol from bit64 (#50015) +* [GH-50022](https://github.com/apache/arrow/issues/50022) - [Dev] Enable auto GitHub Copilot review (#50023) +* [GH-50026](https://github.com/apache/arrow/issues/50026) - [C++][Parquet] SIMD-accelerate SBBF probe via branchless autovec (#50030) +* [GH-50046](https://github.com/apache/arrow/issues/50046) - [CI][C++] Improve caching with apache/infrastructure-actions/stash and more general cache keys (#50047) +* [GH-50052](https://github.com/apache/arrow/issues/50052) - [CI][C++] Bump vcpkg to newest version (#50053) +* [GH-50054](https://github.com/apache/arrow/issues/50054) - [C++][IPC] Validate indices buffer size in ReadSparseCOOIndex (#50055) +* [GH-50057](https://github.com/apache/arrow/issues/50057) - [C++] Avoid signed overflow in Decimal FromString exponent (#50058) +* [GH-50063](https://github.com/apache/arrow/issues/50063) - [C++] Validate buffer size for row-major tensors (#50064) +* [GH-50072](https://github.com/apache/arrow/issues/50072) - [Python] Add tests for replace_with_mask kernel (#50102) +* [GH-50075](https://github.com/apache/arrow/issues/50075) - [C++][Gandiva] fix buffer overrun in to_hex int32/int64 (#50076) +* [GH-50077](https://github.com/apache/arrow/issues/50077) - [C++][IPC] Avoid int64 overflow in ReadSparseCSXIndex (#50038) +* [GH-50078](https://github.com/apache/arrow/issues/50078) - [C++][ORC] Avoid signed overflow when converting timestamps (#50035) +* [GH-50083](https://github.com/apache/arrow/issues/50083) - [C++] Access mimalloc through dynamically-resolved symbols (#41128) +* [GH-50111](https://github.com/apache/arrow/issues/50111) - [C++][Gandiva] Improve function error messages (#50112) +* [GH-50115](https://github.com/apache/arrow/issues/50115) - [Dev] Adjust GitHub Copilot configuration for preliminary reviews (#50117) +* [GH-50139](https://github.com/apache/arrow/issues/50139) - [Dev][Gandiva] Add Gandiva code owners (#50144) +* [GH-50161](https://github.com/apache/arrow/issues/50161) - [C++][IPC] Validate CSF sparse index buffer counts (#50070) +* [GH-50162](https://github.com/apache/arrow/issues/50162) - [C++][Parquet] Avoid int32 overflow in BitPackedRunDecoder::GetBatch offset (#50089) +* [GH-50170](https://github.com/apache/arrow/issues/50170) - [CI][Packaging][Linux] Fix cache (#50185) +* [GH-50172](https://github.com/apache/arrow/issues/50172) - [Doc][Format] Clarify that variadic buffers can also be null (#50255) +* [GH-50182](https://github.com/apache/arrow/issues/50182) - [C++][Parquet] Fix truncated min/max statistics for all-infinity floating-point columns (#50183) +* [GH-50184](https://github.com/apache/arrow/issues/50184) - [C++][Parquet] Avoid reading past truncated statistics values in FormatStatValue (#50025) +* [GH-50189](https://github.com/apache/arrow/issues/50189) - [C++][CI] Remove Ceph install step (#50190) +* [GH-50191](https://github.com/apache/arrow/issues/50191) - [CI][Python] Switch caching to apache/infrastructure-actions/stash (#50192) +* [GH-50197](https://github.com/apache/arrow/issues/50197) - [C++][Python] Add "hypot" compute kernel (#50198) +* [GH-50200](https://github.com/apache/arrow/issues/50200) - [Packaging][Debian] Drop support for bookworm (#50201) +* [GH-50208](https://github.com/apache/arrow/issues/50208) - [CI][C++][Python] Disable ccache `hash_dir` (#50209) +* [GH-50216](https://github.com/apache/arrow/issues/50216) - [C++][Parquet] Add RleBitPackedToBitmapDecoder (#50217) +* [GH-50219](https://github.com/apache/arrow/issues/50219) - [R] Fix duckdb test for dbplyr 2.6.0 (#50220) +* [GH-50225](https://github.com/apache/arrow/issues/50225) - [Ruby] Move merge implementation to ColumnContainable (#50226) +* [GH-50231](https://github.com/apache/arrow/issues/50231) - [C++] Handle unset Substrait extension mapping type (#50263) +* [GH-50236](https://github.com/apache/arrow/issues/50236) - Remove obsolete OpenSUSE 15.5 workarounds (#50258) +* [GH-50237](https://github.com/apache/arrow/issues/50237) - [C++] Migrate arrow/ipc/metadata_internal.h to Result (#50245) +* [GH-50260](https://github.com/apache/arrow/issues/50260) - [C++] Add ComputeLogicalNullCount to ChunkedArray (#50261) +* [GH-50265](https://github.com/apache/arrow/issues/50265) - [C++][Parquet] Update parquet.thrift to sync with 2.13.0 (#50266) +* [GH-50267](https://github.com/apache/arrow/issues/50267) - [C++][Parquet] Upgrade thrift compiler from 0.21.0 to 0.23.0 (#50268) +* [GH-50275](https://github.com/apache/arrow/issues/50275) - [C++][CSV] avoid int32 overflow in block parser value counts (#50074) +* [GH-50283](https://github.com/apache/arrow/issues/50283) - [CI][Ruby] Switch caching to apache/infrastructure-actions/stash (#50284) +* [GH-50292](https://github.com/apache/arrow/issues/50292) - [C++][Parquet] Avoid int64 overflow in CheckReadRangeOrThrow (#50060) +* [GH-50304](https://github.com/apache/arrow/issues/50304) - [C++][IPC] Reject negative sparse tensor shape and non-zero length (#50305) + + + # Apache Arrow 6.0.1 (2021-11-18) ## Bug Fixes From 381fab63a8428c2c0eff90a1d66d7f9f0cb3be07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 3 Jul 2026 09:48:10 +0200 Subject: [PATCH 07/15] MINOR: [Release] Update .deb/.rpm changelogs for 25.0.0 --- .../linux-packages/apache-arrow-apt-source/debian/changelog | 6 ++++++ .../apache-arrow-release/yum/apache-arrow-release.spec.in | 3 +++ dev/tasks/linux-packages/apache-arrow/debian/changelog | 6 ++++++ dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 3 +++ 4 files changed, 18 insertions(+) diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog index e52cc96a2dbf..74414789b8c0 100644 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog @@ -1,3 +1,9 @@ +apache-arrow-apt-source (25.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Fri, 03 Jul 2026 07:48:10 -0000 + apache-arrow-apt-source (24.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in index 5cf507859e10..c243ebc5a75f 100644 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in +++ b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in @@ -85,6 +85,9 @@ else fi %changelog +* Fri Jul 03 2026 Raúl Cumplido - 25.0.0-1 +- New upstream release. + * Tue Apr 14 2026 Raúl Cumplido - 24.0.0-1 - New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/changelog b/dev/tasks/linux-packages/apache-arrow/debian/changelog index 5929ea8cca78..2dd4a3641253 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow/debian/changelog @@ -1,3 +1,9 @@ +apache-arrow (25.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Fri, 03 Jul 2026 07:48:10 -0000 + apache-arrow (24.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 97143a65d394..0eb529f3f171 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -886,6 +886,9 @@ Documentation for Apache Parquet GLib. %endif %changelog +* Fri Jul 03 2026 Raúl Cumplido - 25.0.0-1 +- New upstream release. + * Tue Apr 14 2026 Raúl Cumplido - 24.0.0-1 - New upstream release. From 59bea6ec485e7fe351d1aa6753f964f6a6bc353a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 3 Jul 2026 09:48:16 +0200 Subject: [PATCH 08/15] MINOR: [Release] Update versions for 25.0.0 --- c_glib/meson.build | 2 +- c_glib/vcpkg.json | 2 +- ci/scripts/PKGBUILD | 2 +- cpp/CMakeLists.txt | 2 +- cpp/meson.build | 2 +- cpp/vcpkg.json | 2 +- dev/tasks/homebrew-formulae/apache-arrow-glib.rb | 2 +- dev/tasks/homebrew-formulae/apache-arrow.rb | 2 +- docs/source/_static/versions.json | 9 +++++++-- matlab/CMakeLists.txt | 2 +- python/CMakeLists.txt | 2 +- python/pyproject.toml | 2 +- r/DESCRIPTION | 2 +- r/NEWS.md | 2 +- r/pkgdown/assets/versions.html | 5 +++-- r/pkgdown/assets/versions.json | 8 ++++++-- ruby/red-arrow-cuda/lib/arrow-cuda/version.rb | 2 +- ruby/red-arrow-dataset/lib/arrow-dataset/version.rb | 2 +- .../red-arrow-flight-sql/lib/arrow-flight-sql/version.rb | 2 +- ruby/red-arrow-flight/lib/arrow-flight/version.rb | 2 +- ruby/red-arrow-format/lib/arrow-format/version.rb | 2 +- ruby/red-arrow/lib/arrow/version.rb | 2 +- ruby/red-gandiva/lib/gandiva/version.rb | 2 +- ruby/red-parquet/lib/parquet/version.rb | 2 +- 24 files changed, 37 insertions(+), 27 deletions(-) diff --git a/c_glib/meson.build b/c_glib/meson.build index 6cd615312f50..cdcf052454c7 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -32,7 +32,7 @@ project( # * 22.04: 0.61.2 # * 24.04: 1.3.2 meson_version: '>=0.61.2', - version: '25.0.0-SNAPSHOT', + version: '25.0.0', ) version = meson.project_version() diff --git a/c_glib/vcpkg.json b/c_glib/vcpkg.json index e7919df65b36..ffb42b8e1f3a 100644 --- a/c_glib/vcpkg.json +++ b/c_glib/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrow-glib", - "version-string": "25.0.0-SNAPSHOT", + "version-string": "25.0.0", "$comment:dependencies": "We can enable gobject-introspection again once it's updated", "dependencies": [ "glib", diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 8c025a6a25f9..9fa352f264df 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -18,7 +18,7 @@ _realname=arrow pkgbase=mingw-w64-${_realname} pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}" -pkgver=24.0.0.9000 +pkgver=25.0.0 pkgrel=8000 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)" arch=("any") diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e9f4b6d916bc..e04c112d93c4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -96,7 +96,7 @@ if(POLICY CMP0170) cmake_policy(SET CMP0170 NEW) endif() -set(ARROW_VERSION "25.0.0-SNAPSHOT") +set(ARROW_VERSION "25.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}") diff --git a/cpp/meson.build b/cpp/meson.build index 8158adcaf0d7..2cf481d86e8c 100644 --- a/cpp/meson.build +++ b/cpp/meson.build @@ -19,7 +19,7 @@ project( 'arrow', 'cpp', 'c', - version: '25.0.0-SNAPSHOT', + version: '25.0.0', license: 'Apache-2.0', meson_version: '>=1.3.0', default_options: ['c_std=c11', 'warning_level=2', 'cpp_std=c++20'], diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 1773c216b5c0..5e76f60e55c1 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrow", - "version-string": "25.0.0-SNAPSHOT", + "version-string": "25.0.0", "dependencies": [ "abseil", { diff --git a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb index de38408a5dc6..6062dc40b346 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb @@ -29,7 +29,7 @@ class ApacheArrowGlib < Formula desc "GLib bindings for Apache Arrow" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0-SNAPSHOT/apache-arrow-25.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0/apache-arrow-25.0.0.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index 00a0ef21939b..d5f8f5679270 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -29,7 +29,7 @@ class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0-SNAPSHOT/apache-arrow-25.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0/apache-arrow-25.0.0.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index 435ea8ef9f04..c7e9aa3bf3de 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -1,15 +1,20 @@ [ { - "name": "25.0 (dev)", + "name": "26.0 (dev)", "version": "dev/", "url": "https://arrow.apache.org/docs/dev/" }, { - "name": "24.0 (stable)", + "name": "25.0 (stable)", "version": "", "url": "https://arrow.apache.org/docs/", "preferred": true }, + { + "name": "24.0", + "version": "24.0/", + "url": "https://arrow.apache.org/docs/24.0/" + }, { "name": "23.0", "version": "23.0/", diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 5be5dcba39e0..47d56b9b262d 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -100,7 +100,7 @@ endfunction() set(CMAKE_CXX_STANDARD 20) -set(MLARROW_VERSION "25.0.0-SNAPSHOT") +set(MLARROW_VERSION "25.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MLARROW_BASE_VERSION "${MLARROW_VERSION}") project(mlarrow VERSION "${MLARROW_BASE_VERSION}") diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 1225a1140f3c..44f7854c3045 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -28,7 +28,7 @@ project(pyarrow) # which in turn meant that Py_GIL_DISABLED was not set. set(CMAKE_NO_SYSTEM_FROM_IMPORTED ON) -set(PYARROW_VERSION "25.0.0-SNAPSHOT") +set(PYARROW_VERSION "25.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}") # Generate SO version and full SO version diff --git a/python/pyproject.toml b/python/pyproject.toml index 0054e1a26828..b18f74e3e12c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -107,7 +107,7 @@ root = '..' version_file = 'pyarrow/_generated_version.py' version_scheme = 'guess-next-dev' git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' -fallback_version = '25.0.0a0' +fallback_version = '25.0.0' # TODO: Enable type checking once stubs are merged [tool.mypy] diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 99a1ff318845..0d2a743dd00b 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -1,6 +1,6 @@ Package: arrow Title: Integration to 'Apache' 'Arrow' -Version: 24.0.0.9000 +Version: 25.0.0 Authors@R: c( person("Neal", "Richardson", email = "neal.p.richardson@gmail.com", role = c("aut")), person("Ian", "Cook", email = "ianmcook@gmail.com", role = c("aut")), diff --git a/r/NEWS.md b/r/NEWS.md index b80639fd0e3d..f90d94d1bacb 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -17,7 +17,7 @@ under the License. --> -# arrow 24.0.0.9000 +# arrow 25.0.0 # arrow 24.0.0 diff --git a/r/pkgdown/assets/versions.html b/r/pkgdown/assets/versions.html index b66357ec165a..69b1e75fedfd 100644 --- a/r/pkgdown/assets/versions.html +++ b/r/pkgdown/assets/versions.html @@ -1,7 +1,8 @@ -

24.0.0.9000 (dev)

-

24.0.0 (release)

+

25.0.0.9000 (dev)

+

25.0.0 (release)

+

24.0.0

23.0.1

22.0.0

21.0.0

diff --git a/r/pkgdown/assets/versions.json b/r/pkgdown/assets/versions.json index 2fd5f6b05cdc..29f95e742b35 100644 --- a/r/pkgdown/assets/versions.json +++ b/r/pkgdown/assets/versions.json @@ -1,12 +1,16 @@ [ { - "name": "24.0.0.9000 (dev)", + "name": "25.0.0.9000 (dev)", "version": "dev/" }, { - "name": "24.0.0 (release)", + "name": "25.0.0 (release)", "version": "" }, + { + "name": "24.0.0", + "version": "24.0/" + }, { "name": "23.0.1", "version": "23.0/" diff --git a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb index 62cd8d9e40eb..5207d09980e9 100644 --- a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb +++ b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowCUDA - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb index 4b255e7b12ad..4b4e0d77adc6 100644 --- a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb +++ b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowDataset - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb index babf047e67b5..e230978541d2 100644 --- a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb +++ b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlightSQL - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight/lib/arrow-flight/version.rb b/ruby/red-arrow-flight/lib/arrow-flight/version.rb index c3e0a064f04e..886d0b451f22 100644 --- a/ruby/red-arrow-flight/lib/arrow-flight/version.rb +++ b/ruby/red-arrow-flight/lib/arrow-flight/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlight - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-format/lib/arrow-format/version.rb b/ruby/red-arrow-format/lib/arrow-format/version.rb index f61c89f1fe34..1e319d1ffd73 100644 --- a/ruby/red-arrow-format/lib/arrow-format/version.rb +++ b/ruby/red-arrow-format/lib/arrow-format/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFormat - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow/lib/arrow/version.rb b/ruby/red-arrow/lib/arrow/version.rb index b28b7aa57cf4..719317f1ef09 100644 --- a/ruby/red-arrow/lib/arrow/version.rb +++ b/ruby/red-arrow/lib/arrow/version.rb @@ -16,7 +16,7 @@ # under the License. module Arrow - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-gandiva/lib/gandiva/version.rb b/ruby/red-gandiva/lib/gandiva/version.rb index dd2dad6e0555..8f4da7bb04f6 100644 --- a/ruby/red-gandiva/lib/gandiva/version.rb +++ b/ruby/red-gandiva/lib/gandiva/version.rb @@ -16,7 +16,7 @@ # under the License. module Gandiva - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-parquet/lib/parquet/version.rb b/ruby/red-parquet/lib/parquet/version.rb index 8c23e7f66b86..75393c016831 100644 --- a/ruby/red-parquet/lib/parquet/version.rb +++ b/ruby/red-parquet/lib/parquet/version.rb @@ -16,7 +16,7 @@ # under the License. module Parquet - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") From 8fb632a2eebdaf5ff230a797e587a4c3359f9fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 6 Jul 2026 10:24:12 +0200 Subject: [PATCH 09/15] GH-50383: [Release] Remove deprecated -f flag on conda create environment (#50384) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change The Windows Wheels verification job is currently failing due to a change of API on the conda create call. ### What changes are included in this PR? Remove the deprecated `-f` flag. ### Are these changes tested? Yes, I have pushed the branch to the Apache Arrow repository instead of my fork to be able to manually trigger the workflow for the RC via workflow dispatch in order to validate the changes. ### Are there any user-facing changes? No * GitHub Issue: #50383 Authored-by: Raúl Cumplido Signed-off-by: Raúl Cumplido --- dev/release/verify-release-candidate-wheels.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate-wheels.bat b/dev/release/verify-release-candidate-wheels.bat index df503f67d21a..431e96f24dd8 100644 --- a/dev/release/verify-release-candidate-wheels.bat +++ b/dev/release/verify-release-candidate-wheels.bat @@ -78,7 +78,7 @@ set PY_VERSION_NO_PERIOD=%PY_VERSION:.=% set CONDA_ENV_PATH=%_VERIFICATION_DIR%\_verify-wheel-%PY_VERSION% call conda create -p %CONDA_ENV_PATH% ^ - --no-shortcuts -f -q -y python=%PY_VERSION% ^ + --no-shortcuts -q -y python=%PY_VERSION% ^ || EXIT /B 1 call activate %CONDA_ENV_PATH% From fe141358b2783f2b85eaa982834aece0cb90c5b1 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 21 Jul 2026 11:03:07 +0200 Subject: [PATCH 10/15] GH-50428: [C++] Better mimalloc configuration on macOS (#50549) ### Rationale for this change On macOS, two independently-linked mimalloc v3 instances built with default TLS settings can end up using the same hard-coded TLS slots and crash due to conflicting expectations. See upstream issue at https://github.com/microsoft/mimalloc/issues/1327 This can manifest when PyArrow is loaded side-by-side with another Python extension module that bundles its own instance of mimalloc. ### What changes are included in this PR? 1. Bump mimalloc to 3.4.1, for the availability of the required CMake option. 2. Configure macOS to use C thread-local variables for thread-local storage, avoiding conflicting accesses to hard-coded TLS slots on macOS. 3. Also, unrelatedly, make sure the default malloc is not overriden by our mimalloc build on macOS. ### Are these changes tested? By existing CI jobs. ### Are there any user-facing changes? No, just a bugfix. * GitHub Issue: #50428 Lead-authored-by: Antoine Pitrou Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 12 ++++++++++++ cpp/thirdparty/versions.txt | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index bcbf2e4645a6..1b5030c07d63 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2565,7 +2565,11 @@ if(ARROW_MIMALLOC) "-DCMAKE_C_FLAGS=${MIMALLOC_C_FLAGS}" "-DCMAKE_INSTALL_PREFIX=${MIMALLOC_PREFIX}" -DMI_INSTALL_TOPLEVEL=ON + # Don't override default malloc -DMI_OVERRIDE=OFF + -DMI_OSX_INTERPOSE=OFF + -DMI_OSX_ZONE=OFF + # Allow usage through dlopen (i.e. when libarrow.so itself is dlopen'ed) -DMI_LOCAL_DYNAMIC_TLS=ON -DMI_BUILD_OBJECT=OFF -DMI_BUILD_SHARED=OFF @@ -2573,6 +2577,14 @@ if(ARROW_MIMALLOC) # GH-47229: Force mimalloc to generate armv8.0 binary -DMI_NO_OPT_ARCH=ON) + if(APPLE) + list(APPEND + MIMALLOC_CMAKE_ARGS + # GH-50428: Make sure several mimalloc instances can cohabit in the same process + # (also https://github.com/microsoft/mimalloc/issues/1327#issuecomment-4964140817) + -DMI_TLS_MODEL_LOCAL=ON) + endif() + externalproject_add(mimalloc_ep ${EP_COMMON_OPTIONS} URL ${MIMALLOC_SOURCE_URL} diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index c6f4b01a717c..ff05b6bbae93 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -80,8 +80,8 @@ ARROW_JEMALLOC_BUILD_VERSION=5.3.0 ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa ARROW_LZ4_BUILD_VERSION=v1.10.0 ARROW_LZ4_BUILD_SHA256_CHECKSUM=537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b -ARROW_MIMALLOC_BUILD_VERSION=v3.3.1 -ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=42c16914168ac6741eeb407e83b93a12b2b7ee25a7e14e6b4807fab8b577a540 +ARROW_MIMALLOC_BUILD_VERSION=v3.4.1 +ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=37107a52c16baa80c5f74861dddda7b27bb9949e41a6637691867a94c88ca446 ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.12.0 ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=4b92eb0c06d10683f7447ce9406cb97cd4b453be18d7279320f7b2f025c10187 ARROW_OPENTELEMETRY_BUILD_VERSION=v1.21.0 From d1eaaedef3c7b05668e620fee1478fc884f36296 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 23 Jul 2026 00:15:05 +0800 Subject: [PATCH 11/15] GH-50326: [Python] Convert arrays to Python objects without per-element Scalars in to_pylist (#50327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change `pa.Array.to_pylist()` converts one element at a time through `Array::GetScalar` plus a Python `Scalar` wrapper; for list types each row additionally allocates a Python `Array` wrapper for the row's values slice and a fresh generator before recursing per element. A `sample` profile shows ~20% of runtime in CPython GC (triggered by the per-row GC-tracked allocations), ~25% in `GetScalar`, and only ~7% doing the useful work of creating the output objects — making `to_pylist` several times slower than converting via `to_pandas()` and back, and ~24x slower than `ndarray.tolist()` for plain int64. Details in #50326; this hit Apache Spark's Arrow-serialized Python UDFs (apache/spark#56940, apache/spark#56943). ### What changes are included in this PR? Following review feedback, this adds a general scalar-free conversion mechanism instead of per-type `to_pylist` overrides: - `Array` gains `cdef object _getitem_py(self, int64_t i)`, returning `self[i]` as a Python object. The base implementation is `GetScalar` + `Scalar.as_py`, so any type without a specialization behaves exactly as today (dates, times, timestamps, durations, decimals, dictionary, extension, unions, views, ...). - The baseline `Array.to_pylist` becomes a single loop over `_getitem_py`. `maps_as_pydicts != None` keeps the Scalar-based path, since map→dict conversion has per-entry duplicate-key semantics. - Specializations avoid all per-element Scalar and per-row Array-wrapper allocation: - integers and floats (a `type_id` switch on `NumericArray`; date/time/timestamp subclasses fall through to the exact base), - boolean, - string/binary and large variants (`GetValue` + `PyUnicode_DecodeUTF8` / `PyBytes_FromStringAndSize`, matching `str(buf, 'utf8')` / `to_pybytes()` exactly), - list/large_list/fixed_size_list (each row's list is built from the child's `_getitem_py` over the offset range; the wrapped child is cached on the parent array), - map (association list of key/value tuples, matching `MapScalar.as_py`), - struct (one dict per row; duplicate field names fall back to the Scalar path so they raise `ValueError` like `StructScalar.as_py`). Nested types compose without any per-row wrappers. `ChunkedArray.to_pylist`, `Table.to_pylist` and `ListScalar.as_py` delegate here and speed up automatically. Follow-up candidates: string/binary views, run-end-encoded, dictionary, a fast path for date32. Benchmarks (macOS arm64, M4 Max): | benchmark | before | after | speedup | |---|---|---|---| | flat `int64` with nulls (4M) | 0.39 s | 0.028 s | 14x (~7 ns/element, on par with `ndarray.tolist`) | | flat `string` (4M) | 0.83 s | 0.06 s | 14x | | `list` (2M rows) | 1.93 s | 0.46 s | 4.2x | | `list>` (1M rows) | 2.10 s | 0.40 s | 5.2x | | `struct` (1M rows) | 0.91 s | 0.07 s | 13x | | `map` (1M rows) | 2.77 s | 0.74 s | 3.8x | ### Are these changes tested? `test_to_pylist_bulk_paths` (added here) compares against the per-scalar conversion with exact element types for representative arrays including sliced views. Additionally verified with a randomized differential test against `[x.as_py() for x in arr]` with exact-type equality: all integer widths (incl. values beyond 2^62), floats (NaN/inf), boolean, string/binary (+large, multibyte), all list kinds, nested lists, struct (incl. empty struct, duplicate-field-name `ValueError`), map (incl. strict-mode duplicate-key `KeyError`), dictionary/null fallbacks, sliced/chunked arrays, and both `maps_as_pydicts` modes — no differences. `pytest test_array.py test_scalars.py test_convert_builtin.py test_table.py test_types.py`: 1295 passed. ### Are there any user-facing changes? No behavior changes, only performance. * GitHub Issue: #50326 This pull request and its description were written by Isaac. Lead-authored-by: Liang-Chi Hsieh Co-authored-by: Isaac Signed-off-by: Antoine Pitrou --- python/pyarrow/array.pxi | 157 ++++++++++++++++++++++++++- python/pyarrow/includes/libarrow.pxd | 90 ++++++++------- python/pyarrow/lib.pxd | 7 ++ python/pyarrow/tests/test_array.py | 58 ++++++++++ 4 files changed, 270 insertions(+), 42 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5fc74969abfd..26bb6482a44a 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -1864,7 +1864,24 @@ cdef class Array(_PandasConvertible): lst : list """ self._assert_cpu() - return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + cdef int64_t i, n = self.length() + if maps_as_pydicts is not None: + # Converting maps to dicts has per-entry semantics (duplicate-key + # detection); use the Scalar-based conversion for exact behavior. + # TODO(GH-50429): this falls back to the Scalar path for the whole + # array even when the type contains no maps; threading + # maps_as_pydicts through _getitem_py keeps the fast paths instead. + return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + # TODO(GH-50448): convert per range instead of per element to cut + # the per-element call overhead further. + return [self._getitem_py(i) for i in range(n)] + + cdef object _getitem_py(self, int64_t i): + # Return self[i] as a Python object, without creating a Python Scalar + # (nor, for nested types, per-row Array wrappers) where a subclass + # provides a specialization; this base implementation goes through + # Scalar.as_py and thus preserves its semantics exactly (see GH-50326). + return self.getitem(i).as_py() def tolist(self): """ @@ -2444,6 +2461,12 @@ cdef class BooleanArray(Array): """ Concrete class for Arrow arrays of boolean data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + return ( self.ap).Value(i) + @property def false_count(self): return ( self.ap).false_count() @@ -2458,6 +2481,34 @@ cdef class NumericArray(Array): A base class for Arrow numeric arrays. """ + cdef object _getitem_py(self, int64_t i): + cdef Type tid = self.ap.type_id() + if self.ap.IsNull(i): + return None + if tid == _Type_INT8: + return ( self.ap).Value(i) + elif tid == _Type_INT16: + return ( self.ap).Value(i) + elif tid == _Type_INT32: + return ( self.ap).Value(i) + elif tid == _Type_INT64: + return ( self.ap).Value(i) + elif tid == _Type_UINT8: + return ( self.ap).Value(i) + elif tid == _Type_UINT16: + return ( self.ap).Value(i) + elif tid == _Type_UINT32: + return ( self.ap).Value(i) + elif tid == _Type_UINT64: + return ( self.ap).Value(i) + elif tid == _Type_FLOAT: + return ( self.ap).Value(i) + elif tid == _Type_DOUBLE: + return ( self.ap).Value(i) + # Subclasses whose as_py returns non-primitive objects (dates, times, + # timestamps, durations, half floats, ...) use the exact Scalar path. + return Array._getitem_py(self, i) + cdef class IntegerArray(NumericArray): """ @@ -2776,6 +2827,16 @@ cdef class ListArray(BaseListArray): Concrete class for Arrow arrays of a list data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -2961,6 +3022,16 @@ cdef class LargeListArray(BaseListArray): Identical to ListArray, but 64-bit offsets. """ + cdef object _getitem_py(self, int64_t i): + cdef CLargeListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3551,6 +3622,19 @@ cdef class MapArray(ListArray): Concrete class for Arrow arrays of a map data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = (self.keys, self.items) + cdef Array keys = ( self._children_cache)[0] + cdef Array items = ( self._children_cache)[1] + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + # Matches MapScalar.as_py with the default maps_as_pydicts=None: + # an association list of (key, value) tuples. + return [(keys._getitem_py(j), items._getitem_py(j)) for j in range(start, end)] + @staticmethod def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3688,6 +3772,16 @@ cdef class FixedSizeListArray(BaseListArray): Concrete class for Arrow arrays of a fixed size list data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CFixedSizeListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(values, list_size=None, DataType type=None, mask=None): """ @@ -3974,6 +4068,13 @@ cdef class StringArray(Array): Concrete class for Arrow arrays of string (or utf8) data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + # Matches StringScalar.as_py, which is str(buf, 'utf8'). + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, @@ -4006,6 +4107,12 @@ cdef class LargeStringArray(Array): Concrete class for Arrow arrays of large string (or utf8) data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, @@ -4038,11 +4145,24 @@ cdef class StringViewArray(Array): Concrete class for Arrow arrays of string (or utf8) view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) + cdef class BinaryArray(Array): """ Concrete class for Arrow arrays of variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) + @property def total_values_length(self): """ @@ -4056,6 +4176,13 @@ cdef class LargeBinaryArray(Array): """ Concrete class for Arrow arrays of large variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) + @property def total_values_length(self): """ @@ -4070,6 +4197,12 @@ cdef class BinaryViewArray(Array): Concrete class for Arrow arrays of variable-sized binary view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) + cdef class DictionaryArray(Array): """ @@ -4229,6 +4362,28 @@ cdef class StructArray(Array): Concrete class for Arrow arrays of a struct data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef int64_t k, num_fields = self.type.num_fields + if self._children_cache is None: + names = [self.type.field(k).name for k in range(num_fields)] + if len(set(names)) != len(names): + # Matches StructScalar.as_py + raise ValueError( + "Converting to Python dictionary is not supported when " + "duplicate field names are present") + self._children_cache = ( + names, [self.field(k) for k in range(num_fields)]) + names = ( self._children_cache)[0] + fields = ( self._children_cache)[1] + cdef Array field_arr + result = {} + for k in range(num_fields): + field_arr = fields[k] + result[names[k]] = field_arr._getitem_py(i) + return result + def field(self, index): """ Retrieves the child array belonging to field. diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 8b4786ecbf13..e57c6d0d92d4 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -264,7 +264,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string Diff(const CArray& other) c_bool Equals(const CArray& arr) - c_bool IsNull(int i) + c_bool IsNull(int64_t i) shared_ptr[CArrayData] data() @@ -675,87 +675,87 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string* result) cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray): - c_bool Value(int i) + c_bool Value(int64_t i) int64_t false_count() int64_t true_count() cdef cppclass CUInt8Array" arrow::UInt8Array"(CArray): - uint8_t Value(int i) + uint8_t Value(int64_t i) cdef cppclass CInt8Array" arrow::Int8Array"(CArray): - int8_t Value(int i) + int8_t Value(int64_t i) cdef cppclass CUInt16Array" arrow::UInt16Array"(CArray): - uint16_t Value(int i) + uint16_t Value(int64_t i) cdef cppclass CInt16Array" arrow::Int16Array"(CArray): - int16_t Value(int i) + int16_t Value(int64_t i) cdef cppclass CUInt32Array" arrow::UInt32Array"(CArray): - uint32_t Value(int i) + uint32_t Value(int64_t i) cdef cppclass CInt32Array" arrow::Int32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CUInt64Array" arrow::UInt64Array"(CArray): - uint64_t Value(int i) + uint64_t Value(int64_t i) cdef cppclass CInt64Array" arrow::Int64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CDate32Array" arrow::Date32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CDate64Array" arrow::Date64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CTime32Array" arrow::Time32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CTime64Array" arrow::Time64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CTimestampArray" arrow::TimestampArray"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CDurationArray" arrow::DurationArray"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CMonthDayNanoIntervalArray \ "arrow::MonthDayNanoIntervalArray"(CArray): pass cdef cppclass CHalfFloatArray" arrow::HalfFloatArray"(CArray): - uint16_t Value(int i) + uint16_t Value(int64_t i) cdef cppclass CFloatArray" arrow::FloatArray"(CArray): - float Value(int i) + float Value(int64_t i) cdef cppclass CDoubleArray" arrow::DoubleArray"(CArray): - double Value(int i) + double Value(int64_t i) cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray): - const uint8_t* GetValue(int i) + const uint8_t* GetValue(int64_t i) cdef cppclass CDecimal32Array" arrow::Decimal32Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal64Array" arrow::Decimal64Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal128Array" arrow::Decimal128Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal256Array" arrow::Decimal256Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CListArray" arrow::ListArray"(CArray): @staticmethod @@ -776,8 +776,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: ) const int32_t* raw_value_offsets() - int32_t value_offset(int i) - int32_t value_length(int i) + int32_t value_offset(int64_t i) + int32_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CDataType] value_type() @@ -800,8 +800,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CBuffer] null_bitmap ) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CDataType] value_type() @@ -819,8 +819,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CDataType], shared_ptr[CBuffer] null_bitmap) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CDataType] value_type() @@ -850,8 +850,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const int32_t* raw_value_offsets() const int32_t* raw_value_sizes() - int32_t value_offset(int i) - int32_t value_length(int i) + int32_t value_offset(int64_t i) + int32_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CArray] sizes() @@ -881,8 +881,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CMemoryPool* pool ) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CArray] sizes() @@ -911,8 +911,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CArray] keys() shared_ptr[CArray] items() CMapType* map_type() - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CDataType] value_type() @@ -941,18 +941,20 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const vector[c_string]& field_names, const vector[int8_t]& type_codes) - int32_t value_offset(int i) + int32_t value_offset(int64_t i) shared_ptr[CBuffer] value_offsets() cdef cppclass CBinaryArray" arrow::BinaryArray"(CArray): - const uint8_t* GetValue(int i, int32_t* length) + const uint8_t* GetValue(int64_t i, int32_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int32_t value_offset(int64_t i) int32_t value_length(int64_t i) int32_t total_values_length() cdef cppclass CLargeBinaryArray" arrow::LargeBinaryArray"(CArray): - const uint8_t* GetValue(int i, int64_t* length) + const uint8_t* GetValue(int64_t i, int64_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int64_t value_offset(int64_t i) int64_t value_length(int64_t i) @@ -965,7 +967,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int64_t null_count, int64_t offset) - c_string GetString(int i) + c_string GetString(int64_t i) cdef cppclass CLargeStringArray" arrow::LargeStringArray" \ (CLargeBinaryArray): @@ -975,7 +977,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int64_t null_count, int64_t offset) - c_string GetString(int i) + c_string GetString(int64_t i) + + cdef cppclass CBinaryViewArray" arrow::BinaryViewArray"(CArray): + cpp_string_view GetView(int64_t i) + + cdef cppclass CStringViewArray" arrow::StringViewArray"(CBinaryViewArray): + pass cdef cppclass CStructArray" arrow::StructArray"(CArray): CStructArray(shared_ptr[CDataType]& type, int64_t length, diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 683faa7855c5..38f1ac69a807 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -288,8 +288,15 @@ cdef class Array(_PandasConvertible): # To allow Table to propagate metadata to pandas.Series object _name + cdef: + # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326). + # Appended after the pre-existing attributes to keep their offsets + # stable for extensions compiled against an older pyarrow. + object _children_cache + cdef void init(self, const shared_ptr[CArray]& sp_array) except * cdef getitem(self, int64_t i) + cdef object _getitem_py(self, int64_t i) cdef int64_t length(self) cdef void _assert_cpu(self) except * diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index adc3e097b54a..c1e3f0128be8 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -465,6 +465,64 @@ def test_array_getitem_numpy_scalars(): assert arr[np.int32(idx)].as_py() == lst[idx] +def test_to_pylist_bulk_paths(): + # GH-50326: to_pylist converts through scalar-free _getitem_py + # specializations; the result must match the per-scalar conversion + # exactly. + arrays = [ + pa.array([[1, None, 3], None, [], [4]], type=pa.list_(pa.int32())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.list_(pa.string())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.large_list(pa.large_string())), + pa.array([[1, None], None, [3, 4]], type=pa.list_(pa.int32(), 2)), + pa.array([[[1], [2, None]], None, [None, [3]]], + type=pa.list_(pa.list_(pa.int32()))), + pa.array([[("k1", 1), ("k2", None)], None, []], + type=pa.map_(pa.string(), pa.int32())), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.string()), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.large_string()), + pa.array([b"a\x00b", None, b"", b"\xff"], type=pa.binary()), + pa.array([b"a\x00b", None, b""], type=pa.large_binary()), + # View types store short values inline and long values out-of-line; + # cover both, plus NUL bytes and non-ASCII data. + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}", + "long string exceeding the inline view size"], + type=pa.string_view()), + pa.array([b"a\x00b", None, b"", b"\xff", + b"long binary value exceeding the inline view size"], + type=pa.binary_view()), + pa.array([[b"x", None, b"\x00y"], None, []], + type=pa.list_(pa.binary())), + pa.array([1, None, -(2**62), 2**62], type=pa.int64()), + pa.array([0, None, 2**63 + 7], type=pa.uint64()), + pa.array([-128, 127, None], type=pa.int8()), + pa.array([1.5, None, -0.5], type=pa.float64()), + pa.array([1.5, None], type=pa.float32()), + pa.array([True, None, False], type=pa.bool_()), + pa.array([{"a": 1, "b": "x"}, None, {"a": None, "b": None}], + type=pa.struct([("a", pa.int32()), ("b", pa.string())])), + pa.array([], type=pa.list_(pa.int32())), + pa.array([None, None], type=pa.list_(pa.string())), + ] + for arr in arrays: + for view in (arr, arr.slice(1), arr.slice(0, 2), arr.slice(2)): + assert view.to_pylist() == [x.as_py() for x in view] + + # Values inside numeric lists must stay Python ints/None, never floats + result = pa.array([[1, None, 3]], type=pa.list_(pa.int32())).to_pylist() + assert result == [[1, None, 3]] + assert [type(x) for x in result[0]] == [int, type(None), int] + + # Duplicate struct field names raise like StructScalar.as_py does + dup = pa.StructArray.from_arrays( + [pa.array([1, 2]), pa.array(["a", "b"])], names=["x", "x"]) + with pytest.raises(ValueError, match="duplicate field names"): + dup.to_pylist() + + def test_array_slice(): arr = pa.array(range(10)) From 149158986337257eed43392ee04a2680504de77c Mon Sep 17 00:00:00 2001 From: Antoine Prouvost Date: Thu, 30 Jul 2026 15:05:45 +0200 Subject: [PATCH 12/15] GH-50503: [Parquet] Remove SVE128 unpack (#50611) ### Rationale for this change The SVE128 code path has conflict with the SVE256 that we do not yet manage properly. - There was first the ODR violation in GH-49921 - Now it seems that there may also be an issue with LTO Anyhow, after we fixed the inlining issue in Neon, the SVE128 had no clear advantages over Neon as expected, os this was due to be removed anyways. ### What changes are included in this PR? Remove SVE128 unpack ### Are these changes tested? In CI. ### Are there any user-facing changes? No * GitHub Issue: #50503 Lead-authored-by: AntoinePrv Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/CMakeLists.txt | 28 +++++++++-- cpp/src/arrow/util/bpacking.cc | 7 +-- cpp/src/arrow/util/bpacking_benchmark.cc | 5 -- cpp/src/arrow/util/bpacking_simd_128_alt.cc | 51 --------------------- cpp/src/arrow/util/bpacking_simd_internal.h | 28 ----------- cpp/src/arrow/util/bpacking_test.cc | 9 ---- 6 files changed, 28 insertions(+), 100 deletions(-) delete mode 100644 cpp/src/arrow/util/bpacking_simd_128_alt.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 8750598f6c3b..c8c1de1c85d3 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -346,21 +346,42 @@ endmacro() macro(append_runtime_sve128_src SRCS SRC) if(ARROW_HAVE_RUNTIME_SVE128) list(APPEND ${SRCS} ${SRC}) - set_source_files_properties(${SRC} PROPERTIES COMPILE_OPTIONS "${ARROW_SVE128_FLAGS}") + set(_flags ${ARROW_SVE128_FLAGS}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Disable LTO to work around GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121412 + list(APPEND ${_flags} "-fno-lto") + endif() + set_property(SOURCE ${SRC} + APPEND + PROPERTY COMPILE_OPTIONS ${_flags}) endif() endmacro() macro(append_runtime_sve256_src SRCS SRC) if(ARROW_HAVE_RUNTIME_SVE256) list(APPEND ${SRCS} ${SRC}) - set_source_files_properties(${SRC} PROPERTIES COMPILE_OPTIONS "${ARROW_SVE256_FLAGS}") + set(_flags ${ARROW_SVE256_FLAGS}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Disable LTO to work around GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121412 + list(APPEND ${_flags} "-fno-lto") + endif() + set_property(SOURCE ${SRC} + APPEND + PROPERTY COMPILE_OPTIONS ${_flags}) endif() endmacro() macro(append_runtime_sve512_src SRCS SRC) if(ARROW_HAVE_RUNTIME_SVE512) list(APPEND ${SRCS} ${SRC}) - set_source_files_properties(${SRC} PROPERTIES COMPILE_OPTIONS "${ARROW_SVE512_FLAGS}") + set(_flags ${ARROW_SVE512_FLAGS}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Disable LTO to work around GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121412 + list(APPEND ${_flags} "-fno-lto") + endif() + set_property(SOURCE ${SRC} + APPEND + PROPERTY COMPILE_OPTIONS ${_flags}) endif() endmacro() @@ -588,7 +609,6 @@ append_runtime_avx2_src(ARROW_UTIL_SRCS util/byte_stream_split_internal_avx2.cc) append_runtime_avx2_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc) append_runtime_avx512_src(ARROW_UTIL_SRCS util/bpacking_simd_avx512.cc) -append_runtime_sve128_src(ARROW_UTIL_SRCS util/bpacking_simd_128_alt.cc) append_runtime_sve256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc) if(ARROW_WITH_BROTLI) diff --git a/cpp/src/arrow/util/bpacking.cc b/cpp/src/arrow/util/bpacking.cc index 1bf81df4f28f..b29927345691 100644 --- a/cpp/src/arrow/util/bpacking.cc +++ b/cpp/src/arrow/util/bpacking.cc @@ -32,9 +32,10 @@ struct UnpackDynamicFunction { static constexpr auto targets() { return std::array{ - ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar) // - ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon) // - ARROW_DISPATCH_TARGET_SVE128(&bpacking::unpack_sve128) // + ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar) // + ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon) // + // GH-50503: No SVE128 dispatch as it increases code size without + // increasing performance vs. Neon, and can produce ODR violations. ARROW_DISPATCH_TARGET_SVE256(&bpacking::unpack_sve256) // ARROW_DISPATCH_TARGET_SSE4_2(&bpacking::unpack_sse4_2) // ARROW_DISPATCH_TARGET_AVX2(&bpacking::unpack_avx2) // diff --git a/cpp/src/arrow/util/bpacking_benchmark.cc b/cpp/src/arrow/util/bpacking_benchmark.cc index 93d7cdf165c1..025b493c5927 100644 --- a/cpp/src/arrow/util/bpacking_benchmark.cc +++ b/cpp/src/arrow/util/bpacking_benchmark.cc @@ -206,11 +206,6 @@ BENCHMARK_UNPACK_ALL_TYPES_RUNTIME(Avx512Unaligned, false, bpacking::unpack_avx5 BENCHMARK_UNPACK_ALL_TYPES(NeonUnaligned, false, bpacking::unpack_neon); #endif -#if defined(ARROW_HAVE_RUNTIME_SVE128) -BENCHMARK_UNPACK_ALL_TYPES_RUNTIME(Sve128Unaligned, false, bpacking::unpack_sve128, - SVE128, "Sve128 not available"); -#endif - #if defined(ARROW_HAVE_RUNTIME_SVE256) BENCHMARK_UNPACK_ALL_TYPES_RUNTIME(Sve256Unaligned, false, bpacking::unpack_sve256, SVE256, "Sve256 not available"); diff --git a/cpp/src/arrow/util/bpacking_simd_128_alt.cc b/cpp/src/arrow/util/bpacking_simd_128_alt.cc deleted file mode 100644 index bd4799d3cd31..000000000000 --- a/cpp/src/arrow/util/bpacking_simd_128_alt.cc +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#if defined(ARROW_HAVE_RUNTIME_SVE128) -# define UNPACK_PLATFORM unpack_sve128 -# define KERNEL_PLATFORM KernelSve128 -#endif - -#if !defined(UNPACK_PLATFORM) -# error "This file must be compiled with a known SIMD micro architecture" -#endif - -#include - -#include "arrow/util/bpacking_dispatch_internal.h" -#include "arrow/util/bpacking_simd_internal.h" -#include "arrow/util/bpacking_simd_kernel_internal.h" - -namespace arrow::internal::bpacking { - -template -using KERNEL_PLATFORM = Kernel>; - -template -void UNPACK_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts) { - return unpack_jump(in, out, opts); -} - -template void UNPACK_PLATFORM(const uint8_t*, bool*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint8_t*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint16_t*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint32_t*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint64_t*, const UnpackOptions&); - -} // namespace arrow::internal::bpacking - -#undef UNPACK_PLATFORM diff --git a/cpp/src/arrow/util/bpacking_simd_internal.h b/cpp/src/arrow/util/bpacking_simd_internal.h index d5a81baaec09..78aaa4a8f92a 100644 --- a/cpp/src/arrow/util/bpacking_simd_internal.h +++ b/cpp/src/arrow/util/bpacking_simd_internal.h @@ -53,34 +53,6 @@ extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128( #endif // UNPACK_ARCH128 #undef UNPACK_ARCH128 -#if defined(ARROW_HAVE_RUNTIME_SVE128) -# define UNPACK_ARCH128_ALT unpack_sve128 -#endif - -#if defined(UNPACK_ARCH128_ALT) - -template -ARROW_EXPORT void UNPACK_ARCH128_ALT(const uint8_t* in, Uint* out, - const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( // - const uint8_t* in, bool* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint8_t* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint16_t* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint32_t* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint64_t* out, const UnpackOptions& opts); - -#endif // UNPACK_ARCH128_ALT -#undef UNPACK_ARCH128_ALT - #if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256) # define UNPACK_ARCH256 unpack_sve256 #elif defined(UNPACK_ARCH256) || defined(ARROW_HAVE_RUNTIME_AVX2) diff --git a/cpp/src/arrow/util/bpacking_test.cc b/cpp/src/arrow/util/bpacking_test.cc index d4d588228e7e..0503a15110b6 100644 --- a/cpp/src/arrow/util/bpacking_test.cc +++ b/cpp/src/arrow/util/bpacking_test.cc @@ -301,15 +301,6 @@ TYPED_TEST(TestUnpack, UnpackAvx512) { TYPED_TEST(TestUnpack, UnpackNeon) { this->TestAll(&bpacking::unpack_neon); } #endif -#if defined(ARROW_HAVE_RUNTIME_SVE128) -TYPED_TEST(TestUnpack, UnpackSve128) { - if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE128)) { - GTEST_SKIP() << "Test requires SVE128"; - } - this->TestAll(&bpacking::unpack_sve128); -} -#endif - #if defined(ARROW_HAVE_RUNTIME_SVE256) TYPED_TEST(TestUnpack, UnpackSve256) { if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256)) { From f5037be3f02318738713500f83fc6aef37002f1b Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 30 Jul 2026 18:06:22 -0700 Subject: [PATCH 13/15] GH-50578: [C++][FlightRPC][ODBC] Always return SQL_NO_DATA from GetMoreResults (#50700) ### Rationale for this change Fixes a bug in the implementation of ODBC `GetMoreResults` in the FlightSQL ODBC driver. According to https://learn.microsoft.com/en-us/sql/odbc/reference/appendixes/statement-transitions?view=sql-server-ver17#sqlmoreresults, we should return `SQL_NO_DATA` for some states we previously were throwing another error in. This appears to be exposed by a behavior of only the Windows ODBC driver manager: `GetMoreResults` always gets called even for metadata queries. ### What changes are included in this PR? - Changed implementation and test: `GetMoreResults` now always returns `SQL_NO_DATA`. ### Are these changes tested? Yes, in CI. ### Are there any user-facing changes? No. * GitHub Issue: #50578 Authored-by: Bryce Mecum Signed-off-by: Bryce Mecum --- cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc | 6 +----- cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc | 7 ------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc index 51152c64782e..8b40abfb67d6 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc @@ -784,11 +784,7 @@ SQLRETURN ODBCStatement::GetData(SQLSMALLINT record_number, SQLSMALLINT c_type, SQLRETURN ODBCStatement::GetMoreResults() { // Multiple result sets are not supported by Arrow protocol. - if (current_result_) { - return SQL_NO_DATA; - } else { - throw DriverException("Function sequence error", "HY010"); - } + return SQL_NO_DATA; } void ODBCStatement::GetColumnCount(SQLSMALLINT* column_count_ptr) { diff --git a/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc b/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc index 237626c27820..ba8b883aac47 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc @@ -1984,14 +1984,7 @@ TYPED_TEST(StatementTest, TestSQLMoreResultsNoData) { } TYPED_TEST(StatementTest, TestSQLMoreResultsWithoutQuery) { -#ifdef __linux__ ASSERT_EQ(SQL_NO_DATA, SQLMoreResults(this->stmt)); -#else // Windows & Mac - // Verify function sequence error state is reported when SQLMoreResults is called - // without executing any queries - ASSERT_EQ(SQL_ERROR, SQLMoreResults(this->stmt)); - VerifyOdbcErrorState(SQL_HANDLE_STMT, this->stmt, kErrorStateHY010); -#endif } TYPED_TEST(StatementTest, TestSQLNativeSqlReturnsInputString) { From e70ebf50f0b7a67f3892a307d1dc583e51411bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 4 Aug 2026 12:11:19 +0200 Subject: [PATCH 14/15] GH-50600: [Release] Pin Python version in Conda verification environment --- dev/release/verify-release-candidate.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index ae49367ba6fa..fea869500972 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -362,7 +362,7 @@ install_conda() { maybe_setup_conda() { # Optionally setup conda environment with the passed dependencies local env="conda-${CONDA_ENV:-source}" - local pyver=${PYTHON_VERSION:-3} + local pyver=${PYTHON_VERSION:-3.12} if [ "${USE_CONDA}" -gt 0 ]; then show_info "Configuring Conda environment..." @@ -377,9 +377,10 @@ maybe_setup_conda() { if ! conda env list | cut -d" " -f 1 | grep $env; then mamba create -y -n $env python=${pyver} fi - # Install dependencies + # Install dependencies. Python version pinned so an unversioned + # python dependency does not replace it if [ $# -gt 0 ]; then - mamba install -y -n $env $@ + mamba install -y -n $env python=${pyver} $@ fi # Activate the environment conda activate $env From 849d4634e7784d568477f5e159a13b003a7dff4c Mon Sep 17 00:00:00 2001 From: tadeja Date: Tue, 4 Aug 2026 19:39:55 +0200 Subject: [PATCH 15/15] GH-50808: [Python] Narrow Feather deprecation to V1 format (#50685) ### Rationale for this change Follow-up to https://github.com/apache/arrow/issues/49232#issuecomment-5105290467 / #49590 ### What changes are included in this PR? Remove warnings from `write_feather()`, `read_feather()`, `read_table()`, and `FeatherDataset`. Warn **only** on writing with `version=1` / reading Feather V1 file. Update docs to V1-only and keep IPC migration guide. Additionally corrected notes like `deprecated as of 24.0.0` to `25.0.0` instead. `DeprecationWarning` in place of current `FutureWarning`. ### Are these changes tested? Yes, by CI. ### Are there any user-facing changes? Yes! Feather V2 APIs no longer emit deprecation warnings. Reading/writing the legacy Feather V1 format emits `DeprecationWarning` in place of `FutureWarning`. * GitHub Issue: #50808 Lead-authored-by: Tadeja Kadunc Co-authored-by: tadeja Co-authored-by: Rok Mihevc Signed-off-by: Rok Mihevc --- docs/source/python/api/formats.rst | 7 +-- docs/source/python/feather.rst | 22 +++++---- python/pyarrow/feather.py | 70 +++++++++------------------- python/pyarrow/tests/test_dataset.py | 6 +-- python/pyarrow/tests/test_feather.py | 47 ++++++++++--------- 5 files changed, 66 insertions(+), 86 deletions(-) diff --git a/docs/source/python/api/formats.rst b/docs/source/python/api/formats.rst index 0d2cd8975cde..57a5e824fab1 100644 --- a/docs/source/python/api/formats.rst +++ b/docs/source/python/api/formats.rst @@ -42,14 +42,11 @@ CSV Files .. _api.feather: -Feather Files (Deprecated) --------------------------- +Feather Files +------------- .. currentmodule:: pyarrow.feather -.. deprecated:: 24.0.0 - The Feather API is deprecated. Use the :ref:`IPC ` API instead. - .. autosummary:: :toctree: ../generated/ diff --git a/docs/source/python/feather.rst b/docs/source/python/feather.rst index 76520e912b67..33f8e76bf048 100644 --- a/docs/source/python/feather.rst +++ b/docs/source/python/feather.rst @@ -22,10 +22,6 @@ Feather File Format =================== -.. deprecated:: 24.0.0 - The ``pyarrow.feather`` module is deprecated. Feather V2 is the Arrow IPC - file format. Use :mod:`pyarrow.ipc` instead. See :ref:`ipc` for details. - Feather is a portable file format for storing Arrow tables or data frames (from languages like Python or R) that utilizes the :ref:`Arrow IPC format ` internally. Feather was created early in the Arrow project as a proof of @@ -39,8 +35,8 @@ R. There are two file format versions for Feather: * Version 1 (V1), a legacy version available starting in 2016, replaced by V2. V1 files are distinct from Arrow IPC files and lack many features, such as the ability to store all Arrow data types. V1 files also lack compression - support. We intend to maintain read support for V1 for the foreseeable - future. + support. Reading and writing V1 files is deprecated as of 25.0.0 and will + be removed in a future version. The ``pyarrow.feather`` module contains the read and write functions for the format. :func:`~pyarrow.feather.write_feather` accepts either a @@ -108,13 +104,23 @@ reduced disk IO requirements. Writing Version 1 (V1) Files ---------------------------- +.. deprecated:: 25.0.0 + Support for the legacy Feather V1 format is deprecated. Reading and + writing V1 files will be removed in a future version. Rewrite V1 files + in the Arrow IPC file format (Feather V2). + For compatibility with libraries without support for Version 2 files, you can -write the version 1 format by passing ``version=1`` to ``write_feather``. We -intend to maintain read support for V1 for the foreseeable future. +write the version 1 format by passing ``version=1`` to ``write_feather``. Migration to IPC ---------------- +.. note:: + + ``pyarrow.feather.write_feather`` and ``pyarrow.feather.read_table`` + equivalents will be provided in :mod:`pyarrow.ipc` before the + ``pyarrow.feather`` module is deprecated. + Since Feather V2 is the Arrow IPC file format, you can use the :mod:`pyarrow.ipc` module as a direct replacement: diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py index 68f708c91489..60d59b0e0b66 100644 --- a/python/pyarrow/feather.py +++ b/python/pyarrow/feather.py @@ -32,9 +32,6 @@ class FeatherDataset: """ Encapsulates details of reading a list of Feather files. - .. deprecated:: 24.0.0 - Use :func:`pyarrow.dataset.dataset` with ``format='ipc'`` instead. - Parameters ---------- path_or_paths : List[str] @@ -44,12 +41,6 @@ class FeatherDataset: """ def __init__(self, path_or_paths, validate_schema=True): - warnings.warn( - "pyarrow.feather.FeatherDataset is deprecated as of 24.0.0. " - "Use pyarrow.dataset.dataset() with format='ipc' instead.", - FutureWarning, - stacklevel=2 - ) self.paths = path_or_paths self.validate_schema = validate_schema @@ -127,11 +118,6 @@ def write_feather(df, dest, compression=None, compression_level=None, """ Write a pandas.DataFrame to Feather format. - .. deprecated:: 24.0.0 - Use :func:`pyarrow.ipc.new_file` / - :class:`pyarrow.ipc.RecordBatchFileWriter` instead. - Feather V2 is the Arrow IPC file format. - Parameters ---------- df : pandas.DataFrame or pyarrow.Table @@ -150,15 +136,20 @@ def write_feather(df, dest, compression=None, compression_level=None, which is currently 64K version : int, default 2 Feather file version. Version 2 is the current. Version 1 is the more - limited legacy format + limited legacy format. + + .. deprecated:: 25.0.0 + Writing Feather V1 files is deprecated. Use the default + ``version=2`` to write Arrow IPC files instead. """ - warnings.warn( - "pyarrow.feather.write_feather is deprecated as of 24.0.0. " - "Use pyarrow.ipc.new_file() / RecordBatchFileWriter instead. " - "Feather V2 is the Arrow IPC file format.", - FutureWarning, - stacklevel=2 - ) + if version == 1: + warnings.warn( + "Feather V1 files are deprecated as of 25.0.0 and support will " + "be removed in a future version. Use the default version=2 to " + "write Arrow IPC files instead.", + DeprecationWarning, + stacklevel=2 + ) if _pandas_api.have_pandas: if (_pandas_api.has_sparse and isinstance(df, _pandas_api.pd.SparseDataFrame)): @@ -223,11 +214,6 @@ def read_feather(source, columns=None, use_threads=True, Read a pandas.DataFrame from Feather format. To read as pyarrow.Table use feather.read_table. - .. deprecated:: 24.0.0 - Use :func:`pyarrow.ipc.open_file` / - :class:`pyarrow.ipc.RecordBatchFileReader` instead. - Feather V2 is the Arrow IPC file format. - Parameters ---------- source : str file path, or file-like object @@ -249,13 +235,6 @@ def read_feather(source, columns=None, use_threads=True, df : pandas.DataFrame The contents of the Feather file as a pandas.DataFrame """ - warnings.warn( - "pyarrow.feather.read_feather is deprecated as of 24.0.0. " - "Use pyarrow.ipc.open_file() / RecordBatchFileReader instead. " - "Feather V2 is the Arrow IPC file format.", - FutureWarning, - stacklevel=2 - ) return (_read_table_internal( source, columns=columns, memory_map=memory_map, use_threads=use_threads).to_pandas(use_threads=use_threads, **kwargs)) @@ -265,11 +244,20 @@ def _read_table_internal(source, columns=None, memory_map=False, use_threads=True): """ Internal implementation for reading a Feather file as a pyarrow.Table. - Does not emit deprecation warnings. + Emits a deprecation warning if the file is a legacy Feather V1 file. """ reader = _feather.FeatherReader( source, use_memory_map=memory_map, use_threads=use_threads) + if reader.version < 3: + warnings.warn( + "Feather V1 files are deprecated as of 25.0.0 and support will " + "be removed in a future version. Consider rewriting this file " + "in the Arrow IPC file format (Feather V2).", + DeprecationWarning, + stacklevel=3 + ) + if columns is None: return reader.read() @@ -302,11 +290,6 @@ def read_table(source, columns=None, memory_map=False, use_threads=True): """ Read a pyarrow.Table from Feather format - .. deprecated:: 24.0.0 - Use :func:`pyarrow.ipc.open_file` / - :class:`pyarrow.ipc.RecordBatchFileReader` instead. - Feather V2 is the Arrow IPC file format. - Parameters ---------- source : str file path, or file-like object @@ -324,13 +307,6 @@ def read_table(source, columns=None, memory_map=False, use_threads=True): table : pyarrow.Table The contents of the Feather file as a pyarrow.Table """ - warnings.warn( - "pyarrow.feather.read_table is deprecated as of 24.0.0. " - "Use pyarrow.ipc.open_file() / RecordBatchFileReader instead. " - "Feather V2 is the Arrow IPC file format.", - FutureWarning, - stacklevel=2 - ) return _read_table_internal(source, columns=columns, memory_map=memory_map, use_threads=use_threads) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 63c537eae5fc..74693317d711 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -1927,7 +1927,6 @@ def test_fragments_parquet_subset_with_nested_fields(tempdir): @pytest.mark.pandas @pytest.mark.parquet -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_fragments_repr(tempdir, dataset): # partitioned parquet dataset fragment = list(dataset.get_fragments())[0] @@ -3700,7 +3699,7 @@ def test_column_names_encoding(tempdir, dataset_reader): assert dataset_transcoded.to_table().equals(expected_table) -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") +@pytest.mark.filterwarnings("ignore:Feather V1:DeprecationWarning") def test_feather_format(tempdir, dataset_reader): from pyarrow.feather import write_feather @@ -4082,7 +4081,6 @@ def test_dataset_project_null_column(tempdir, dataset_reader): assert dataset_reader.to_table(dataset).equals(expected) -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_dataset_project_columns(tempdir, dataset_reader): # basic column re-projection with expressions from pyarrow import feather @@ -4434,7 +4432,6 @@ def test_write_dataset_with_dataset(tempdir): @pytest.mark.pandas -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_write_dataset_existing_data(tempdir): directory = tempdir / 'ds' table = pa.table({'b': ['x', 'y', 'z'], 'c': [1, 2, 3]}) @@ -5058,7 +5055,6 @@ def test_write_dataset_arrow_schema_metadata(tempdir): assert result["a"].type.tz == "Europe/Brussels" -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_write_dataset_schema_metadata(tempdir): # ensure that schema metadata gets written from pyarrow import feather diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py index c04cef534ee3..8c9e7eb437e6 100644 --- a/python/pyarrow/tests/test_feather.py +++ b/python/pyarrow/tests/test_feather.py @@ -41,10 +41,10 @@ except ImportError: pass -# Suppress deprecation warnings for existing tests since pyarrow.feather -# is deprecated as of 24.0.0 +# Suppress deprecation warnings for existing tests that intentionally +# exercise the deprecated Feather V1 format pytestmark = pytest.mark.filterwarnings( - "ignore:pyarrow.feather:FutureWarning" + "ignore:Feather V1:DeprecationWarning" ) @@ -894,39 +894,44 @@ def test_feather_datetime_resolution_arrow_to_pandas(tempdir): # --- Deprecation warning tests --- @pytest.mark.pandas -@pytest.mark.filterwarnings("default:pyarrow.feather:FutureWarning") -def test_feather_deprecation_warnings(tempdir): +@pytest.mark.filterwarnings("default:Feather V1:DeprecationWarning") +def test_feather_v1_deprecation_warnings(tempdir): table = pa.table({"a": [1, 2, 3]}) path = str(tempdir / "test.feather") - with pytest.warns(FutureWarning, match="write_feather is deprecated"): - write_feather(table, path) + with pytest.warns(DeprecationWarning, match="Feather V1"): + write_feather(table, path, version=1) - with pytest.warns(FutureWarning, match="read_table is deprecated"): + with pytest.warns(DeprecationWarning, match="Feather V1"): read_table(path) - with pytest.warns(FutureWarning, match="read_feather is deprecated"): + with pytest.warns(DeprecationWarning, match="Feather V1"): read_feather(path) -@pytest.mark.filterwarnings("default:pyarrow.feather:FutureWarning") -def test_feather_dataset_deprecated(): - with pytest.warns(FutureWarning, match="FeatherDataset is deprecated"): - FeatherDataset([]) +def test_feather_v2_no_deprecation_warning(tempdir): + table = pa.table({"a": [1, 2, 3]}) + path = str(tempdir / "test.feather") + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + write_feather(table, path) + read_table(path) + FeatherDataset([path]).read_table() @pytest.mark.pandas -@pytest.mark.filterwarnings("default:pyarrow.feather:FutureWarning") -def test_read_feather_no_double_warning(tempdir): - """Verify read_feather emits exactly one FutureWarning, not two.""" +@pytest.mark.filterwarnings("default:Feather V1:DeprecationWarning") +def test_read_feather_v1_no_double_warning(tempdir): + """Verify reading a V1 file emits one DeprecationWarning, not two.""" table = pa.table({"a": [1, 2, 3]}) path = str(tempdir / "test.feather") with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - write_feather(table, path) + warnings.simplefilter("ignore", DeprecationWarning) + write_feather(table, path, version=1) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") read_feather(path) - future_warnings = [x for x in w if issubclass(x.category, - FutureWarning)] - assert len(future_warnings) == 1 + v1_warnings = [x for x in w if issubclass(x.category, + DeprecationWarning)] + assert len(v1_warnings) == 1