diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 9a863d2a5ff7..8b072ed90afa 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -66,7 +66,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: cpp_extra @@ -80,6 +79,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') diff --git a/.github/workflows/cuda_extra.yml b/.github/workflows/cuda_extra.yml index 64f2597fe89f..f58ee86e6fac 100644 --- a/.github/workflows/cuda_extra.yml +++ b/.github/workflows/cuda_extra.yml @@ -49,7 +49,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: cuda_extra @@ -63,6 +62,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: CUDA') diff --git a/.github/workflows/package_linux.yml b/.github/workflows/package_linux.yml index b5afb7f36a0d..b96627f49ff4 100644 --- a/.github/workflows/package_linux.yml +++ b/.github/workflows/package_linux.yml @@ -58,7 +58,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: package_linux @@ -72,6 +71,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: Package: Linux') diff --git a/.github/workflows/r_extra.yml b/.github/workflows/r_extra.yml index 1ba7d2247cef..142464dabc75 100644 --- a/.github/workflows/r_extra.yml +++ b/.github/workflows/r_extra.yml @@ -61,7 +61,6 @@ permissions: jobs: check-labels: - if: github.event_name == 'pull_request' uses: ./.github/workflows/check_labels.yml with: parent-workflow: r_extra @@ -75,6 +74,7 @@ jobs: steps: - id: set_enabled if: >- + github.ref_type == 'tag' || (github.event_name == 'schedule' && github.repository == 'apache/arrow') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: R') diff --git a/CHANGELOG.md b/CHANGELOG.md index 6101f5d3cac2..3c653e79c318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,235 @@ +# Apache Arrow 25.0.0 (2026-07-07 00:00:00+00:00) + +## Bug Fixes + +* [GH-17081](https://github.com/apache/arrow/issues/17081) - [C++] arrow::stl::TupleRangeFromTable docs incorrect (#50095) +* [GH-20403](https://github.com/apache/arrow/issues/20403) - [Doc] pyarrow.Array.diff Examples is wrongly rendered (#50096) +* [GH-32994](https://github.com/apache/arrow/issues/32994) - [Dev][Archery] Fix multi-arch Docker configuration (#50125) +* [GH-36503](https://github.com/apache/arrow/issues/36503) - [C++] Make DictionaryArray::dictionary() thread-safe (#48905) +* [GH-38558](https://github.com/apache/arrow/issues/38558) - [C++] Add support for null sort option per sort key (#46926) +* [GH-39603](https://github.com/apache/arrow/issues/39603) - [R] Error: Cannot convert Dictionary Array of type dictionary to R (#49710) +* [GH-39754](https://github.com/apache/arrow/issues/39754) - [R] Docs are not clear on expected behaviour of date parsing functions (e.g. dmy()) on Windows vs. Linux/MacOS (#49708) +* [GH-39784](https://github.com/apache/arrow/issues/39784) - [C++][Gandiva] Fix decimal in_expr crash with cached object code (#49951) +* [GH-40410](https://github.com/apache/arrow/issues/40410) - [C++] Skip only s3fs-tests and s3fs-module-tests that require MinIO if MinIO is not available (#50215) +* [GH-40640](https://github.com/apache/arrow/issues/40640) - [R] to_arrow() loses group_by() (#49713) +* [GH-40742](https://github.com/apache/arrow/issues/40742) - [R] fix max_rows_per_group must be a positive number (#49709) +* [GH-40886](https://github.com/apache/arrow/issues/40886) - [R] Cryptic error when creating Arrow array from POSIXct with invalid time zones (#49714) +* [GH-43574](https://github.com/apache/arrow/issues/43574) - [Python][Parquet] do not add partition columns from file path when reading single file (#49853) +* [GH-45193](https://github.com/apache/arrow/issues/45193) - [C++][Compute] Treat NaNs and nulls as distinct values in rank tie-breaking (#49304) +* [GH-46179](https://github.com/apache/arrow/issues/46179) - [Python] Bump index level once if pandas df already contains __index_level_i__ column (#46884) +* [GH-47100](https://github.com/apache/arrow/issues/47100) - [Docs] Correct the Statistics schema specification (#50092) +* [GH-47252](https://github.com/apache/arrow/issues/47252) - [C++][Compute] Fix sort_indices for temporal types in arrow::Table (#50270) +* [GH-47447](https://github.com/apache/arrow/issues/47447) - [C++] Fix replace_with_mask for null type arrays (#49950) +* [GH-47481](https://github.com/apache/arrow/issues/47481) - [C++][Acero] record_batch_reader_source specify Ordering::Implicit to support `select * limit k` (#47482) +* [GH-47642](https://github.com/apache/arrow/issues/47642) - [C++] Catch exceptions from initial_task in AsyncTaskScheduler (#49860) +* [GH-47657](https://github.com/apache/arrow/issues/47657) - [C++][Parquet] Check for integer overflow when coercing timestamps (#49615) +* [GH-48094](https://github.com/apache/arrow/issues/48094) - [C++] Restrict SecureString capacity tail check to Linux (#49906) +* [GH-48254](https://github.com/apache/arrow/issues/48254) - [Python][Parquet] Support extension types in read_schema (#48255) +* [GH-48712](https://github.com/apache/arrow/issues/48712) - [R] "Invalid metadata$r" warning (#49608) +* [GH-48801](https://github.com/apache/arrow/issues/48801) - [C++] Set CMAKE_POLICY_VERSION_MINIMUM for RapidJSON (#49993) +* [GH-48926](https://github.com/apache/arrow/issues/48926) - [C++] Upgrade Abseil/Protobuf/GRPC/Google-Cloud-CPP bundled versions (#48964) +* [GH-49272](https://github.com/apache/arrow/issues/49272) - [C++][CI] Fix intermittent segfault in arrow-json-test with MinGW (#49462) +* [GH-49327](https://github.com/apache/arrow/issues/49327) - [Python][Packaging] Use ARROW_SIMD_LEVEL=NEON for macOS arm64 (#50181) +* [GH-49433](https://github.com/apache/arrow/issues/49433) - [C++] Buffer ARROW_LOG output to prevent thread interleaving (#49663) +* [GH-49465](https://github.com/apache/arrow/issues/49465) - [CI][C++] Fix Abseil hang in arrow-flight-test on ODBC Windows (#50085) +* [GH-49522](https://github.com/apache/arrow/issues/49522) - [CI] Update chrome_version for emscripten job to latest stable (v148) (#49523) +* [GH-49614](https://github.com/apache/arrow/issues/49614) - [C++] Report an error instead of silent truncation in base64_decode on invalid input (#49660) +* [GH-49689](https://github.com/apache/arrow/issues/49689) - [R][C++] Parquets do not support list-columns of ordered factors (ordered dictionaries) (#49937) +* [GH-49719](https://github.com/apache/arrow/issues/49719) - [C++] Rename vendored date header guards (#49778) +* [GH-49740](https://github.com/apache/arrow/issues/49740) - [C++][Python] Fix casts to view types leaving null variadic buffers (#50166) +* [GH-49743](https://github.com/apache/arrow/issues/49743) - [Release] Split the vote thread preparation into its own shell script (#49770) +* [GH-49745](https://github.com/apache/arrow/issues/49745) - [Docs][Python] Fix doctests failure in substrait.rst (#49754) +* [GH-49752](https://github.com/apache/arrow/issues/49752) - [C++][Gandiva] Fix potential buffer overrun in Gandiva SSL function (#49780) +* [GH-49753](https://github.com/apache/arrow/issues/49753) - [C++][Gandiva] Fix overflow in string functions (#49813) +* [GH-49757](https://github.com/apache/arrow/issues/49757) - [Release][CI] export SSL_CERT_FILE to bypass incompatibility with OpenSSL on RHEL-8 (#49769) +* [GH-49759](https://github.com/apache/arrow/issues/49759) - [C++][Integration] Harden BinaryView JSON parsing with runtime validation (#49758) +* [GH-49764](https://github.com/apache/arrow/issues/49764) - [C++][Python] Avoid building bundled Abseil outside resolve_dependency (#49936) +* [GH-49767](https://github.com/apache/arrow/issues/49767) - [CI][C++] Disable mold on Ubuntu 24.04 to work around mold#1247 (#50033) +* [GH-49803](https://github.com/apache/arrow/issues/49803) - [C++][CI] Avoid aborting when fuzz-mutated IPC file has less batches (#49804) +* [GH-49823](https://github.com/apache/arrow/issues/49823) - [CI] Add missing contents: read permission to Package Linux workflow (#49825) +* [GH-49831](https://github.com/apache/arrow/issues/49831) - [Python] Withhold annotations from Python wheel until they are complete (#50168) +* [GH-49834](https://github.com/apache/arrow/issues/49834) - [C++] Avoid building re2 unit tests (#50152) +* [GH-49837](https://github.com/apache/arrow/issues/49837) - [C++][Parquet] Avoid unbounded temporary std::vector in DELTA_(LENGTH_)BYTE_ARRAY decoder (#49838) +* [GH-49846](https://github.com/apache/arrow/issues/49846) - [CI][Python] fix test-conda-python-3.11-hypothesis (#49847) +* [GH-49861](https://github.com/apache/arrow/issues/49861) - [R] Missing R libarrow binary for linux-arm64 (#49893) +* [GH-49866](https://github.com/apache/arrow/issues/49866) - [R][Release] Restore using tzdb on Windows for tzdata (#49867) +* [GH-49872](https://github.com/apache/arrow/issues/49872) - [C++] Remove deprecated std::is_trivial (#49871) +* [GH-49875](https://github.com/apache/arrow/issues/49875) - [Python] Fix timezone dropped when converting tz-aware Categorical to Arrow array (#49878) +* [GH-49888](https://github.com/apache/arrow/issues/49888) - [C++][Compute] Fix count for run-end encoded arrays with nulls (#49908) +* [GH-49896](https://github.com/apache/arrow/issues/49896) - [C++] Reject short buffer reads in IPC reader (#49897) +* [GH-49905](https://github.com/apache/arrow/issues/49905) - [Archery] Fix archery benchmark diff with pandas 3 (#49912) +* [GH-49917](https://github.com/apache/arrow/issues/49917) - [Python] Remove Py_XDECREF to avoid Use-After-Free on `PyList_SetItem` in `SparseCSFTensorToNdarray` (#49916) +* [GH-49923](https://github.com/apache/arrow/issues/49923) - [Parquet][Python] Inconsistent default values for Parquet pre_buffer (#49924) +* [GH-49927](https://github.com/apache/arrow/issues/49927) - [Python][Parquet] Expose bloom_filter_offset and bloom_filter_length to Python in column chunk metadata (#49926) +* [GH-49930](https://github.com/apache/arrow/issues/49930) - [CI][C++] Pin MinGW MSYS2 packages to unblock CI (#49931) +* [GH-49933](https://github.com/apache/arrow/issues/49933) - [Python] Fix test_table_column_subset_metadata to set freq on correct object (#49944) +* [GH-49942](https://github.com/apache/arrow/issues/49942) - [Python] Protect PyBuffer and NumPyBuffer destructors against interpreter finalization (#49943) +* [GH-49948](https://github.com/apache/arrow/issues/49948) - [CI][C++] Revert PR 49931 (Pin MinGW MSYS2 packages) but keep bumped minIO version (#49945) +* [GH-49956](https://github.com/apache/arrow/issues/49956) - [GLib] Add fallback data type for unknown extension data type (#49969) +* [GH-49966](https://github.com/apache/arrow/issues/49966) - [C++] Detect different endianness between IPC file and stream in IPC file fuzzer (#49968) +* [GH-49974](https://github.com/apache/arrow/issues/49974) - [C++][CMake] Add missing result variable initialization for `validate_apple_libtool()` (#49975) +* [GH-49991](https://github.com/apache/arrow/issues/49991) - [C++][FlightRPC] Fix unity build ordering issue (#49990) +* [GH-49994](https://github.com/apache/arrow/issues/49994) - [CI] Remove `brew uninstall pkg-config` workarounds (#49997) +* [GH-49998](https://github.com/apache/arrow/issues/49998) - [CI][Python] Pin to an older release of miniforge to fix mamba hang (#49999) +* [GH-50000](https://github.com/apache/arrow/issues/50000) - [C++][FlightRPC] Use grpcpp/grpcpp.h not grpcpp/version_info.h for old gRPC (#50001) +* [GH-50009](https://github.com/apache/arrow/issues/50009) - [R] FinalizeS3 segfaults for stale connection (#50081) +* [GH-50010](https://github.com/apache/arrow/issues/50010) - [C++][Parquet] Fix undefined behavior in TypedColumnWriterImpl::UpdateLevelHistogram (#50011) +* [GH-50012](https://github.com/apache/arrow/issues/50012) - [Python] Fix list_ storage crashes when values exceed int32 offsets (#50016) +* [GH-50017](https://github.com/apache/arrow/issues/50017) - [CI][Release] Remove deprecated -f (force) flag from conda create on Windows verification job (#50018) +* [GH-50037](https://github.com/apache/arrow/issues/50037) - [Python] test_table_uses_memory_pool flaky on macOS 14 job (#50045) +* [GH-50041](https://github.com/apache/arrow/issues/50041) - [CI][Python] Make test_string_to_tzinfo_pytz_fallback more robust for platforms supporting lower case tz names (#50042) +* [GH-50043](https://github.com/apache/arrow/issues/50043) - [C++][Python] Fix hash_any/hash_all on sliced boolean arrays (#50094) +* [GH-50051](https://github.com/apache/arrow/issues/50051) - [C++][Parquet] Avoid size overflow in WKBBuffer::ReadCoords (#50036) +* [GH-50065](https://github.com/apache/arrow/issues/50065) - [Packaging][CI][C++] Drop unused libboost-system-dev (#50066) +* [GH-50090](https://github.com/apache/arrow/issues/50090) - [Packaging] Avoid building CUDA for Debian forky due to missing nvidia-cuda-toolkit (#50279) +* [GH-50098](https://github.com/apache/arrow/issues/50098) - [C++][Parquet] Enable filesystem support when building Parquet utilities (#50099) +* [GH-50103](https://github.com/apache/arrow/issues/50103) - [C++] Missing iosfwd include in cpp/src/arrow/util/string_util.h (#50101) +* [GH-50105](https://github.com/apache/arrow/issues/50105) - [C++][Python] Fix sliced sparse union null checks (#50108) +* [GH-50109](https://github.com/apache/arrow/issues/50109) - [C++][Gandiva] Fix incorrect error messages (#50110) +* [GH-50113](https://github.com/apache/arrow/issues/50113) - [C++][Python] Fix `count` for sliced union arrays (#50114) +* [GH-50129](https://github.com/apache/arrow/issues/50129) - [C++][Parquet] Enable `ARROW_JSON` automatically with `PARQUET_REQUIRE_ENCRYPTION` (#50130) +* [GH-50133](https://github.com/apache/arrow/issues/50133) - [CI] Make extra labels entirely manual on PRs (#50256) +* [GH-50149](https://github.com/apache/arrow/issues/50149) - [C++][Parquet] Avoid process abort when encoding fuzzer encounters OOM (#50150) +* [GH-50156](https://github.com/apache/arrow/issues/50156) - [C++][Parquet] Ignore min/max for unknown column order (#50157) +* [GH-50163](https://github.com/apache/arrow/issues/50163) - [R] Bug: Partial matching on $metadata$r causes errors with schema metadata keys starting with "r" (#50178) +* [GH-50171](https://github.com/apache/arrow/issues/50171) - [CI][R] Build libarrow is failing for Linux arm64 (#50179) +* [GH-50173](https://github.com/apache/arrow/issues/50173) - [C++][Gandiva] Install tzdata-legacy to fix failing TestTime.TestCastTimestampWithTZ from gandiva/precompiled/time_test.cc (#50211) +* [GH-50174](https://github.com/apache/arrow/issues/50174) - [CI][Python] Fix debug Python job (#50207) +* [GH-50176](https://github.com/apache/arrow/issues/50176) - [Python] Explicitly pass exc_type=ImportError to importorskip pyarrow.* (#50177) +* [GH-50204](https://github.com/apache/arrow/issues/50204) - [C++] Fix S3FileSystem::MakeUri dropping the bucket on Windows MinGW (#50205) +* [GH-50210](https://github.com/apache/arrow/issues/50210) - [C++][Gandiva] Replace precompiled std::string to fix _Unwind_Resume JIT failure on JNI builds (#50214) +* [GH-50212](https://github.com/apache/arrow/issues/50212) - [C++][Parquet] Add PrintTo to fix failure in parquet-internals-test on test-conda-cpp-valgrind (#50213) +* [GH-50218](https://github.com/apache/arrow/issues/50218) - [C++][FlightRPC] Add missing sudo to fix ODBC Linux job (#50307) +* [GH-50227](https://github.com/apache/arrow/issues/50227) - [Docs][Python][Parquet] Update parquet default version text in parquet.rst (#50228) +* [GH-50240](https://github.com/apache/arrow/issues/50240) - [C++] Make IPC message decoding stricter (#50235) +* [GH-50242](https://github.com/apache/arrow/issues/50242) - [CI][Release] Fix flaky dylib load failure in macOS verify-rc build (#50243) +* [GH-50253](https://github.com/apache/arrow/issues/50253) - [CI] Manually install aws-sdk-cpp with brew in order to avoid lock (#50254) +* [GH-50257](https://github.com/apache/arrow/issues/50257) - [CI][Docs] Fix doxygen failing due to double backticks (#50259) +* [GH-50277](https://github.com/apache/arrow/issues/50277) - [CI][Python] Avoid using generators for test parametrization on newer Pytest (#50278) +* [GH-50291](https://github.com/apache/arrow/issues/50291) - [Python][Packaging] Stop using nightly build dependencies for building free-threaded wheels (#50315) +* [GH-50293](https://github.com/apache/arrow/issues/50293) - [CI] Run check-labels for all triggers to avoid cancelling further steps and add tag to set_enabled (#50340) +* [GH-50294](https://github.com/apache/arrow/issues/50294) - [C++][R] Missing typename in ulp_distance.cc breaks clang-15 builds (#50296) +* [GH-50295](https://github.com/apache/arrow/issues/50295) - [C++][R] #include in vector_select_k.cc breaks macOS CRAN and wasm builds (#50297) +* [GH-50300](https://github.com/apache/arrow/issues/50300) - [C++][CI] Update include to generated/Message_generated.h so Meson is able to find (#50301) +* [GH-50318](https://github.com/apache/arrow/issues/50318) - [R][CI] Install missing libpng-dev for test-r-linux-as-cran (#50328) +* [GH-50330](https://github.com/apache/arrow/issues/50330) - [C++][R][Parquet] Add missing typename in RleBitPackedDecoderGetRunDecode (#50332) +* [GH-50336](https://github.com/apache/arrow/issues/50336) - [Release][Archery] Fix archery GitHub integration for release scripts (#50337) + + +## New Features and Improvements + +* [GH-14796](https://github.com/apache/arrow/issues/14796) - [Dev][COMPONENT] " to issue title automatically (#49892) +* [GH-19667](https://github.com/apache/arrow/issues/19667) - [C++][Gandiva] Use arrow::Result for RegexUtil::SqlLikePatternToPcre (#49879) +* [GH-22232](https://github.com/apache/arrow/issues/22232) - [C++][Python] Introduce optional default_column_type parameter (#47663) +* [GH-31318](https://github.com/apache/arrow/issues/31318) - [Python] Add fixed-offset timezones to Hypothesis test strategy (#49844) +* [GH-32381](https://github.com/apache/arrow/issues/32381) - [C++] Improve error handling for hash table merges (#49512) +* [GH-33241](https://github.com/apache/arrow/issues/33241) - [Archery] Replace github3 with pygithub (#48886) +* [GH-33390](https://github.com/apache/arrow/issues/33390) - [R] Field-level metadata (#49631) +* [GH-33420](https://github.com/apache/arrow/issues/33420) - [R] Improve error message when providing a mix of readr and Arrow options (#50048) +* [GH-38849](https://github.com/apache/arrow/issues/38849) - [C++][Parquet] Add support for list view and large list view (#50160) +* [GH-40062](https://github.com/apache/arrow/issues/40062) - [C++][Python] Conversion of Table to Arrow Tensor (#41870) +* [GH-45187](https://github.com/apache/arrow/issues/45187) - [Ruby] Ensure initializing all rb_memory_view_t members (#50234) +* [GH-45331](https://github.com/apache/arrow/issues/45331) - [C++] Use xsimd for CPU feature detection (#49940) +* [GH-45819](https://github.com/apache/arrow/issues/45819) - [C++] Add OptionalBitmapAnd utility (#49848) +* [GH-46178](https://github.com/apache/arrow/issues/46178) - [R] source_node alignment warning (#50120) +* [GH-46369](https://github.com/apache/arrow/issues/46369) - [C++] Add key-value pairs to the FileSystemFactory interface and to S3Options (#50044) +* [GH-46914](https://github.com/apache/arrow/issues/46914) - [C++][FlightSQL] Remove boost/algorithm/string.h dependency (#50241) +* [GH-46956](https://github.com/apache/arrow/issues/46956) - [Docs][CI] Enable version switcher in local and PR preview build (#46957) +* [GH-47435](https://github.com/apache/arrow/issues/47435) - [Python][Parquet] Add direct key encryption/decryption API (#49667) +* [GH-47769](https://github.com/apache/arrow/issues/47769) - [C++] SVE dynamic dispatch (#49756) +* [GH-47876](https://github.com/apache/arrow/issues/47876) - [C++][FlightRPC] ODBC: macOS `.PKG` installer for Intel and ARM (#49766) +* [GH-48028](https://github.com/apache/arrow/issues/48028) - [Python][Packaging] Update quay.io base manylinux and musllinux images for Python wheels and remove cp313t (#50082) +* [GH-48068](https://github.com/apache/arrow/issues/48068) - [C++][FlightRPC] Linux ODBC: Configure Dremio instance to allow remote testing (#49695) +* [GH-48294](https://github.com/apache/arrow/issues/48294) - [Ruby] Add RecordBatch#merge (#50175) +* [GH-48408](https://github.com/apache/arrow/issues/48408) - [C++] Enable ULP-based float comparison (#49290) +* [GH-49232](https://github.com/apache/arrow/issues/49232) - [Python] deprecate feather python (#49590) +* [GH-49275](https://github.com/apache/arrow/issues/49275) - [Doc] Update docs to specify disclosure of AI on mailing list messages (#49277) +* [GH-49497](https://github.com/apache/arrow/issues/49497) - [FlightRPC] Add is_update field to ActionCreatePreparedStatementResult (#49498) +* [GH-49534](https://github.com/apache/arrow/issues/49534) - [R] Implement dplyr recode_values(), replace_values(), and replace_when() (#49536) +* [GH-49537](https://github.com/apache/arrow/issues/49537) - [C++][FlightRPC] Windows CI to Support ODBC DLL & MSI Signing (#49603) +* [GH-49552](https://github.com/apache/arrow/issues/49552) - [C++][FlightRPC][ODBC] Enable ODBC test build on Linux (#49668) +* [GH-49644](https://github.com/apache/arrow/issues/49644) - [Python] Support converting list of multi-dimensional arrays to FixedShapeTensor (#50203) +* [GH-49651](https://github.com/apache/arrow/issues/49651) - [C++][FlightRPC] Fix ODBC Linux test segmentation fault (#49688) +* [GH-49675](https://github.com/apache/arrow/issues/49675) - [Docs] Make "stale issues" more visible to potential contributors and document stale issue policy (#49703) +* [GH-49684](https://github.com/apache/arrow/issues/49684) - [MATLAB] Introduce deprecation warnings for "legacy" Feather V1 functions `featherread` and `featherwrite` (#49705) +* [GH-49686](https://github.com/apache/arrow/issues/49686) - [C++][FlightRPC][ODBC][Release] Create signing script for Windows FlightSQL ODBC build (#49788) +* [GH-49700](https://github.com/apache/arrow/issues/49700) - [R][CI][Dev] Use air precommit hook (#49701) +* [GH-49720](https://github.com/apache/arrow/issues/49720) - [C++] Optimize base64_decode validation using lookup table (#49748) +* [GH-49723](https://github.com/apache/arrow/issues/49723) - [C++][FlightRPC][ODBC] Update ODBC Documentation (#49851) +* [GH-49725](https://github.com/apache/arrow/issues/49725) - [CI] Use environment variables instead of template expressions in workflow run blocks (#49733) +* [GH-49728](https://github.com/apache/arrow/issues/49728) - [CI] Set persist-credentials: false in checkout actions (#49734) +* [GH-49729](https://github.com/apache/arrow/issues/49729) - [CI] Scope workflow permissions and secret inheritance (#49773) +* [GH-49737](https://github.com/apache/arrow/issues/49737) - [R][CI] Disable Rhub GCC 12 and GCC 13 with LTO jobs - images no longer match CRAN (#49739) +* [GH-49738](https://github.com/apache/arrow/issues/49738) - [R][CI] Re-enable GCC+LTO job once rhub has a GCC 15 image (#49795) +* [GH-49751](https://github.com/apache/arrow/issues/49751) - [Python] Add raw fd support to pa.OSFile (#49750) +* [GH-49772](https://github.com/apache/arrow/issues/49772) - [C++] Bump bundled mimalloc version (#49801) +* [GH-49776](https://github.com/apache/arrow/issues/49776) - [CI][C++] Install libc6-dbg in apt-based Linux C++ images (#50034) +* [GH-49783](https://github.com/apache/arrow/issues/49783) - [C++][FlightRPC][ODBC] Reuse connections across test suite (#49784) +* [GH-49785](https://github.com/apache/arrow/issues/49785) - [C++][FlightRPC][ODBC] Get ODBC tests passing on Linux (#49786) +* [GH-49789](https://github.com/apache/arrow/issues/49789) - [C++] Use `CMAKE_INSTALL_DOCDIR` instead of static `share/doc/${PROJECT_NAME}` (#49790) +* [GH-49793](https://github.com/apache/arrow/issues/49793) - [R] Update NEWS.md for 24.0.0 (#49794) +* [GH-49805](https://github.com/apache/arrow/issues/49805) - [C++][Parquet] Avoid unbounded temporary allocation in DeltaBitPackDecoder::DecodeArrow (#49806) +* [GH-49807](https://github.com/apache/arrow/issues/49807) - [CI] Remove obsolete test-ubuntu-22.04-cpp-20 job (#49827) +* [GH-49835](https://github.com/apache/arrow/issues/49835) - [C++] A constexpr dynamic dispatch with static dispatch when possible (#49840) +* [GH-49890](https://github.com/apache/arrow/issues/49890) - [Dev] Group files under component comment headers in `.github/CODEOWNERS` (#49891) +* [GH-49898](https://github.com/apache/arrow/issues/49898) - [C++][CI] Use mold in more builds (#49899) +* [GH-49901](https://github.com/apache/arrow/issues/49901) - [R] Bump minimum supported R version to 4.2 now that 4.6 is out (#49929) +* [GH-49913](https://github.com/apache/arrow/issues/49913) - [Archery] Add preserve-dir and improve directory layout (#50056) +* [GH-49918](https://github.com/apache/arrow/issues/49918) - [C++][Parquet] Catch std::vector allocation errors in encoding fuzzer (#49919) +* [GH-49921](https://github.com/apache/arrow/issues/49921) - [C++] Bump xsimd to 14.2.0 (#49922) +* [GH-49938](https://github.com/apache/arrow/issues/49938) - [C++] Bump bundled c-ares to 1.34.6 (#49939) +* [GH-49946](https://github.com/apache/arrow/issues/49946) - [Format] Better document equivalence between IPC file and streams (#49947) +* [GH-49952](https://github.com/apache/arrow/issues/49952) - [C++][Gandiva] Use timegm in date_time_test utilities (#49953) +* [GH-49959](https://github.com/apache/arrow/issues/49959) - [C++][Parquet] Avoid unbounded temp alloc in BYTE_STREAM_SPLIT decoder (#49960) +* [GH-49967](https://github.com/apache/arrow/issues/49967) - [Python][CI] Raise oldest NumPy wheel-test requirement to a patched release (#49965) +* [GH-49981](https://github.com/apache/arrow/issues/49981) - [R][Packaging] Support building R package under r-universe/r-wasm (#49982) +* [GH-49988](https://github.com/apache/arrow/issues/49988) - [CI][Packaging] Enable reproducible builds on host for APT based Linux packages (#48148) +* [GH-50005](https://github.com/apache/arrow/issues/50005) - [C++] Use FetchContent for RapidJSON (#50006) +* [GH-50007](https://github.com/apache/arrow/issues/50007) - [C++][Parquet] Add bloom filter folding to automatically size SBBF filters (#50008) +* [GH-50014](https://github.com/apache/arrow/issues/50014) - [R] Replace imported symbol from bit64 (#50015) +* [GH-50022](https://github.com/apache/arrow/issues/50022) - [Dev] Enable auto GitHub Copilot review (#50023) +* [GH-50026](https://github.com/apache/arrow/issues/50026) - [C++][Parquet] SIMD-accelerate SBBF probe via branchless autovec (#50030) +* [GH-50046](https://github.com/apache/arrow/issues/50046) - [CI][C++] Improve caching with apache/infrastructure-actions/stash and more general cache keys (#50047) +* [GH-50052](https://github.com/apache/arrow/issues/50052) - [CI][C++] Bump vcpkg to newest version (#50053) +* [GH-50054](https://github.com/apache/arrow/issues/50054) - [C++][IPC] Validate indices buffer size in ReadSparseCOOIndex (#50055) +* [GH-50057](https://github.com/apache/arrow/issues/50057) - [C++] Avoid signed overflow in Decimal FromString exponent (#50058) +* [GH-50063](https://github.com/apache/arrow/issues/50063) - [C++] Validate buffer size for row-major tensors (#50064) +* [GH-50072](https://github.com/apache/arrow/issues/50072) - [Python] Add tests for replace_with_mask kernel (#50102) +* [GH-50075](https://github.com/apache/arrow/issues/50075) - [C++][Gandiva] fix buffer overrun in to_hex int32/int64 (#50076) +* [GH-50077](https://github.com/apache/arrow/issues/50077) - [C++][IPC] Avoid int64 overflow in ReadSparseCSXIndex (#50038) +* [GH-50078](https://github.com/apache/arrow/issues/50078) - [C++][ORC] Avoid signed overflow when converting timestamps (#50035) +* [GH-50083](https://github.com/apache/arrow/issues/50083) - [C++] Access mimalloc through dynamically-resolved symbols (#41128) +* [GH-50111](https://github.com/apache/arrow/issues/50111) - [C++][Gandiva] Improve function error messages (#50112) +* [GH-50115](https://github.com/apache/arrow/issues/50115) - [Dev] Adjust GitHub Copilot configuration for preliminary reviews (#50117) +* [GH-50139](https://github.com/apache/arrow/issues/50139) - [Dev][Gandiva] Add Gandiva code owners (#50144) +* [GH-50161](https://github.com/apache/arrow/issues/50161) - [C++][IPC] Validate CSF sparse index buffer counts (#50070) +* [GH-50162](https://github.com/apache/arrow/issues/50162) - [C++][Parquet] Avoid int32 overflow in BitPackedRunDecoder::GetBatch offset (#50089) +* [GH-50170](https://github.com/apache/arrow/issues/50170) - [CI][Packaging][Linux] Fix cache (#50185) +* [GH-50172](https://github.com/apache/arrow/issues/50172) - [Doc][Format] Clarify that variadic buffers can also be null (#50255) +* [GH-50182](https://github.com/apache/arrow/issues/50182) - [C++][Parquet] Fix truncated min/max statistics for all-infinity floating-point columns (#50183) +* [GH-50184](https://github.com/apache/arrow/issues/50184) - [C++][Parquet] Avoid reading past truncated statistics values in FormatStatValue (#50025) +* [GH-50189](https://github.com/apache/arrow/issues/50189) - [C++][CI] Remove Ceph install step (#50190) +* [GH-50191](https://github.com/apache/arrow/issues/50191) - [CI][Python] Switch caching to apache/infrastructure-actions/stash (#50192) +* [GH-50197](https://github.com/apache/arrow/issues/50197) - [C++][Python] Add "hypot" compute kernel (#50198) +* [GH-50200](https://github.com/apache/arrow/issues/50200) - [Packaging][Debian] Drop support for bookworm (#50201) +* [GH-50208](https://github.com/apache/arrow/issues/50208) - [CI][C++][Python] Disable ccache `hash_dir` (#50209) +* [GH-50216](https://github.com/apache/arrow/issues/50216) - [C++][Parquet] Add RleBitPackedToBitmapDecoder (#50217) +* [GH-50219](https://github.com/apache/arrow/issues/50219) - [R] Fix duckdb test for dbplyr 2.6.0 (#50220) +* [GH-50225](https://github.com/apache/arrow/issues/50225) - [Ruby] Move merge implementation to ColumnContainable (#50226) +* [GH-50231](https://github.com/apache/arrow/issues/50231) - [C++] Handle unset Substrait extension mapping type (#50263) +* [GH-50236](https://github.com/apache/arrow/issues/50236) - Remove obsolete OpenSUSE 15.5 workarounds (#50258) +* [GH-50237](https://github.com/apache/arrow/issues/50237) - [C++] Migrate arrow/ipc/metadata_internal.h to Result (#50245) +* [GH-50260](https://github.com/apache/arrow/issues/50260) - [C++] Add ComputeLogicalNullCount to ChunkedArray (#50261) +* [GH-50265](https://github.com/apache/arrow/issues/50265) - [C++][Parquet] Update parquet.thrift to sync with 2.13.0 (#50266) +* [GH-50267](https://github.com/apache/arrow/issues/50267) - [C++][Parquet] Upgrade thrift compiler from 0.21.0 to 0.23.0 (#50268) +* [GH-50275](https://github.com/apache/arrow/issues/50275) - [C++][CSV] avoid int32 overflow in block parser value counts (#50074) +* [GH-50283](https://github.com/apache/arrow/issues/50283) - [CI][Ruby] Switch caching to apache/infrastructure-actions/stash (#50284) +* [GH-50292](https://github.com/apache/arrow/issues/50292) - [C++][Parquet] Avoid int64 overflow in CheckReadRangeOrThrow (#50060) +* [GH-50304](https://github.com/apache/arrow/issues/50304) - [C++][IPC] Reject negative sparse tensor shape and non-zero length (#50305) + + + # Apache Arrow 6.0.1 (2021-11-18) ## Bug Fixes diff --git a/c_glib/meson.build b/c_glib/meson.build index 6cd615312f50..cdcf052454c7 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -32,7 +32,7 @@ project( # * 22.04: 0.61.2 # * 24.04: 1.3.2 meson_version: '>=0.61.2', - version: '25.0.0-SNAPSHOT', + version: '25.0.0', ) version = meson.project_version() diff --git a/c_glib/vcpkg.json b/c_glib/vcpkg.json index e7919df65b36..ffb42b8e1f3a 100644 --- a/c_glib/vcpkg.json +++ b/c_glib/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrow-glib", - "version-string": "25.0.0-SNAPSHOT", + "version-string": "25.0.0", "$comment:dependencies": "We can enable gobject-introspection again once it's updated", "dependencies": [ "glib", diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 8c025a6a25f9..9fa352f264df 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -18,7 +18,7 @@ _realname=arrow pkgbase=mingw-w64-${_realname} pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}" -pkgver=24.0.0.9000 +pkgver=25.0.0 pkgrel=8000 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)" arch=("any") diff --git a/ci/scripts/r_install_system_dependencies.sh b/ci/scripts/r_install_system_dependencies.sh index 237e0e9408b3..042d6a5c9adb 100755 --- a/ci/scripts/r_install_system_dependencies.sh +++ b/ci/scripts/r_install_system_dependencies.sh @@ -38,10 +38,11 @@ fi # Install curl, OpenSSL, and libuv # - curl/OpenSSL: technically only needed for S3/GCS support, but # installing the R curl package fails without it +# - libpng: required by the png R package # - libuv: required by the fs R package (no longer bundles libuv by default) case "$PACKAGE_MANAGER" in apt-get) - apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev + apt-get install -y libcurl4-openssl-dev libpng-dev libssl-dev libuv1-dev ;; apk) $PACKAGE_MANAGER add curl-dev openssl-dev libuv-dev diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e9f4b6d916bc..e04c112d93c4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -96,7 +96,7 @@ if(POLICY CMP0170) cmake_policy(SET CMP0170 NEW) endif() -set(ARROW_VERSION "25.0.0-SNAPSHOT") +set(ARROW_VERSION "25.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}") diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index bcbf2e4645a6..1b5030c07d63 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2565,7 +2565,11 @@ if(ARROW_MIMALLOC) "-DCMAKE_C_FLAGS=${MIMALLOC_C_FLAGS}" "-DCMAKE_INSTALL_PREFIX=${MIMALLOC_PREFIX}" -DMI_INSTALL_TOPLEVEL=ON + # Don't override default malloc -DMI_OVERRIDE=OFF + -DMI_OSX_INTERPOSE=OFF + -DMI_OSX_ZONE=OFF + # Allow usage through dlopen (i.e. when libarrow.so itself is dlopen'ed) -DMI_LOCAL_DYNAMIC_TLS=ON -DMI_BUILD_OBJECT=OFF -DMI_BUILD_SHARED=OFF @@ -2573,6 +2577,14 @@ if(ARROW_MIMALLOC) # GH-47229: Force mimalloc to generate armv8.0 binary -DMI_NO_OPT_ARCH=ON) + if(APPLE) + list(APPEND + MIMALLOC_CMAKE_ARGS + # GH-50428: Make sure several mimalloc instances can cohabit in the same process + # (also https://github.com/microsoft/mimalloc/issues/1327#issuecomment-4964140817) + -DMI_TLS_MODEL_LOCAL=ON) + endif() + externalproject_add(mimalloc_ep ${EP_COMMON_OPTIONS} URL ${MIMALLOC_SOURCE_URL} diff --git a/cpp/meson.build b/cpp/meson.build index 8158adcaf0d7..2cf481d86e8c 100644 --- a/cpp/meson.build +++ b/cpp/meson.build @@ -19,7 +19,7 @@ project( 'arrow', 'cpp', 'c', - version: '25.0.0-SNAPSHOT', + version: '25.0.0', license: 'Apache-2.0', meson_version: '>=1.3.0', default_options: ['c_std=c11', 'warning_level=2', 'cpp_std=c++20'], diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 8750598f6c3b..c8c1de1c85d3 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -346,21 +346,42 @@ endmacro() macro(append_runtime_sve128_src SRCS SRC) if(ARROW_HAVE_RUNTIME_SVE128) list(APPEND ${SRCS} ${SRC}) - set_source_files_properties(${SRC} PROPERTIES COMPILE_OPTIONS "${ARROW_SVE128_FLAGS}") + set(_flags ${ARROW_SVE128_FLAGS}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Disable LTO to work around GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121412 + list(APPEND ${_flags} "-fno-lto") + endif() + set_property(SOURCE ${SRC} + APPEND + PROPERTY COMPILE_OPTIONS ${_flags}) endif() endmacro() macro(append_runtime_sve256_src SRCS SRC) if(ARROW_HAVE_RUNTIME_SVE256) list(APPEND ${SRCS} ${SRC}) - set_source_files_properties(${SRC} PROPERTIES COMPILE_OPTIONS "${ARROW_SVE256_FLAGS}") + set(_flags ${ARROW_SVE256_FLAGS}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Disable LTO to work around GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121412 + list(APPEND ${_flags} "-fno-lto") + endif() + set_property(SOURCE ${SRC} + APPEND + PROPERTY COMPILE_OPTIONS ${_flags}) endif() endmacro() macro(append_runtime_sve512_src SRCS SRC) if(ARROW_HAVE_RUNTIME_SVE512) list(APPEND ${SRCS} ${SRC}) - set_source_files_properties(${SRC} PROPERTIES COMPILE_OPTIONS "${ARROW_SVE512_FLAGS}") + set(_flags ${ARROW_SVE512_FLAGS}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Disable LTO to work around GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121412 + list(APPEND ${_flags} "-fno-lto") + endif() + set_property(SOURCE ${SRC} + APPEND + PROPERTY COMPILE_OPTIONS ${_flags}) endif() endmacro() @@ -588,7 +609,6 @@ append_runtime_avx2_src(ARROW_UTIL_SRCS util/byte_stream_split_internal_avx2.cc) append_runtime_avx2_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc) append_runtime_avx512_src(ARROW_UTIL_SRCS util/bpacking_simd_avx512.cc) -append_runtime_sve128_src(ARROW_UTIL_SRCS util/bpacking_simd_128_alt.cc) append_runtime_sve256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc) if(ARROW_WITH_BROTLI) diff --git a/cpp/src/arrow/compute/kernels/select_k_test.cc b/cpp/src/arrow/compute/kernels/select_k_test.cc index 67e5d214636b..47e4af58001c 100644 --- a/cpp/src/arrow/compute/kernels/select_k_test.cc +++ b/cpp/src/arrow/compute/kernels/select_k_test.cc @@ -276,6 +276,26 @@ TEST_F(TestSelectKWithArray, FullSelectKNull) { Check(uint8(), array_input, options, expected); } +TEST_F(TestSelectKWithArray, PartialSelectKAllNull) { + auto array_input = R"([null, null, null, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(4, sort_keys); + auto expected = R"([null, null, null, null])"; + Check(uint8(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + Check(uint8(), array_input, options, expected); +} + +TEST_F(TestSelectKWithArray, FullSelectKAllNull) { + auto array_input = R"([null, null, null, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = R"([null, null, null, null, null])"; + Check(uint8(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + Check(uint8(), array_input, options, expected); +} + TEST_F(TestSelectKWithArray, PartialSelectKNullNaN) { auto array_input = R"([null, 30, NaN, 20, 10, null])"; std::vector sort_keys{SortKey("a", SortOrder::Descending)}; @@ -293,9 +313,31 @@ TEST_F(TestSelectKWithArray, FullSelectKNullNaN) { options.sort_keys[0].null_placement = NullPlacement::AtStart; Check(float64(), array_input, options, "[null, null, NaN, 30, 20, 10]"); } + +TEST_F(TestSelectKWithArray, PartialSelectKAllNullAndNaN) { + auto array_input = R"([null, NaN, NaN, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(4, sort_keys); + auto expected = R"([NaN, NaN, null, null])"; + Check(float64(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = R"([null, null, null, NaN])"; + Check(float64(), array_input, options, expected); +} + +TEST_F(TestSelectKWithArray, FullSelectKAllNullAndNaN) { + auto array_input = R"([null, NaN, NaN, null, null])"; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = R"([NaN, NaN, null, null, null])"; + Check(float64(), array_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = R"([null, null, null, NaN, NaN])"; + Check(float64(), array_input, options, expected); +} + // Test basic cases for chunked array -template struct TestSelectKWithChunkedArray : public ::testing::Test { TestSelectKWithChunkedArray() {} @@ -323,6 +365,15 @@ struct TestSelectKWithChunkedArray : public ::testing::Test { AssertSelectK(chunked_array, k); } + void Check(const std::shared_ptr& type, const std::vector& input, + const SelectKOptions& options, const std::string& expected) { + std::shared_ptr actual; + auto input_array = ChunkedArrayFromJSON(type, input); + auto expected_array = ChunkedArrayFromJSON(type, {expected}); + ASSERT_OK(this->DoSelectK(input_array, options, &actual)); + AssertChunkedEqual(*expected_array, *actual); + } + void Check(const std::shared_ptr& chunked_array, const SelectKOptions& options, const std::shared_ptr& expected_array) { @@ -343,9 +394,12 @@ struct TestSelectKWithChunkedArray : public ::testing::Test { } }; -TYPED_TEST_SUITE(TestSelectKWithChunkedArray, SelectKableTypes); +template +struct TestSelectKWithChunkedArrayTyped : public TestSelectKWithChunkedArray {}; + +TYPED_TEST_SUITE(TestSelectKWithChunkedArrayTyped, SelectKableTypes); -TYPED_TEST(TestSelectKWithChunkedArray, RandomValuesWithSlices) { +TYPED_TEST(TestSelectKWithChunkedArrayTyped, RandomValuesWithSlices) { Random rand(0x61549225); int length = 100; for (auto null_probability : {0.0, 0.1, 0.5, 1.0}) { @@ -361,62 +415,119 @@ TYPED_TEST(TestSelectKWithChunkedArray, RandomValuesWithSlices) { } } -TYPED_TEST(TestSelectKWithChunkedArray, PartialSelectKNull) { - auto chunked_array = ChunkedArrayFromJSON(uint8(), { - "[null, 1]", - "[3, null, 2]", - "[1]", - }); +TEST_F(TestSelectKWithChunkedArray, PartialSelectKNull) { + auto chunked_array = std::vector{ + "[null, 1]", + "[3, null, 2]", + "[1]", + }; std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; auto options = SelectKOptions(3, sort_keys); - auto expected = ChunkedArrayFromJSON(uint8(), {"[1, 1, 2]"}); - this->Check(chunked_array, options, expected); + auto expected = "[1, 1, 2]"; + this->Check(uint8(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtStart; - expected = ChunkedArrayFromJSON(uint8(), {"[null, null, 1]"}); - this->Check(chunked_array, options, expected); + expected = "[null, null, 1]"; + this->Check(uint8(), chunked_array, options, expected); } -TYPED_TEST(TestSelectKWithChunkedArray, FullSelectKNull) { - auto chunked_array = ChunkedArrayFromJSON(uint8(), { - "[null, 1]", - "[3, null, 2]", - "[1]", - }); +TEST_F(TestSelectKWithChunkedArray, FullSelectKNull) { + auto chunked_array = std::vector{ + "[null, 1]", + "[3, null, 2]", + "[1]", + }; std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; auto options = SelectKOptions(10, sort_keys); options.sort_keys[0].null_placement = NullPlacement::AtStart; - auto expected = ChunkedArrayFromJSON(uint8(), {"[null, null, 1, 1, 2, 3]"}); - this->Check(chunked_array, options, expected); + auto expected = "[null, null, 1, 1, 2, 3]"; + this->Check(uint8(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtEnd; - expected = ChunkedArrayFromJSON(uint8(), {"[1, 1, 2, 3, null, null]"}); - this->Check(chunked_array, options, expected); + expected = "[1, 1, 2, 3, null, null]"; + this->Check(uint8(), chunked_array, options, expected); } -TYPED_TEST(TestSelectKWithChunkedArray, PartialSelectKNullNaN) { - auto chunked_array = ChunkedArrayFromJSON( - float64(), {"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}); +TEST_F(TestSelectKWithChunkedArray, PartialSelectKAllNull) { + auto chunked_array = std::vector{ + "[null, null]", + "[null, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(3, sort_keys); + auto expected = "[null, null, null]"; + this->Check(uint8(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + this->Check(uint8(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, FullSelectKAllNull) { + auto chunked_array = std::vector{ + "[null, null]", + "[null, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = "[null, null, null, null, null, null]"; + this->Check(uint8(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + this->Check(uint8(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, PartialSelectKNullNaN) { + auto chunked_array = + std::vector{"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}; std::vector sort_keys{SortKey("a", SortOrder::Descending)}; auto options = SelectKOptions(3, sort_keys); options.sort_keys[0].null_placement = NullPlacement::AtStart; - auto expected = ChunkedArrayFromJSON(float64(), {"[null, null, NaN]"}); - this->Check(chunked_array, options, expected); + auto expected = "[null, null, NaN]"; + this->Check(float64(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtEnd; - expected = ChunkedArrayFromJSON(float64(), {"[10, 3, 2]"}); - this->Check(chunked_array, options, expected); + expected = "[10, 3, 2]"; + this->Check(float64(), chunked_array, options, expected); } -TYPED_TEST(TestSelectKWithChunkedArray, FullSelectKNullNaN) { - auto chunked_array = ChunkedArrayFromJSON( - float64(), {"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}); +TEST_F(TestSelectKWithChunkedArray, FullSelectKNullNaN) { + auto chunked_array = + std::vector{"[null, 1]", "[3, null, NaN]", "[10, NaN, 2]", "[1]"}; std::vector sort_keys{SortKey("a", SortOrder::Descending)}; auto options = SelectKOptions(10, sort_keys); options.sort_keys[0].null_placement = NullPlacement::AtStart; - auto expected = - ChunkedArrayFromJSON(float64(), {"[null, null, NaN, NaN, 10, 3, 2, 1, 1]"}); - this->Check(chunked_array, options, expected); + auto expected = "[null, null, NaN, NaN, 10, 3, 2, 1, 1]"; + this->Check(float64(), chunked_array, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtEnd; - expected = ChunkedArrayFromJSON(float64(), {"[10, 3, 2, 1, 1, NaN, NaN, null, null]"}); - this->Check(chunked_array, options, expected); + expected = "[10, 3, 2, 1, 1, NaN, NaN, null, null]"; + this->Check(float64(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, PartialSelectKAllNullAndNaN) { + auto chunked_array = std::vector{ + "[null, NaN]", + "[NaN, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(3, sort_keys); + auto expected = "[NaN, NaN, null]"; + this->Check(float64(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = "[null, null, null]"; + this->Check(float64(), chunked_array, options, expected); +} + +TEST_F(TestSelectKWithChunkedArray, FullSelectKAllNullAndNaN) { + auto chunked_array = std::vector{ + "[null, NaN]", + "[NaN, null, null]", + "[null]", + }; + std::vector sort_keys{SortKey("a", SortOrder::Ascending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = "[NaN, NaN, null, null, null, null]"; + this->Check(float64(), chunked_array, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = "[null, null, null, null, NaN, NaN]"; + this->Check(float64(), chunked_array, options, expected); } template @@ -790,6 +901,39 @@ TEST_F(TestSelectKWithRecordBatch, PartialSelectKNullNaN) { Check(schema, batch_input, options, expected); } +TEST_F(TestSelectKWithRecordBatch, PartialSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + auto batch_input = R"([ + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN}, + {"a": NaN, "b": NaN}, + {"a": null, "b": NaN} + ])"; + std::vector sort_keys{ + SortKey("a", SortOrder::Ascending, NullPlacement::AtStart), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(3, sort_keys); + auto expected = R"([{"a": null, "b": NaN}, + {"a": null, "b": NaN}, + {"a": null, "b": null}])"; + Check(schema, batch_input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}])"; + Check(schema, batch_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtEnd; + expected = R"([{"a": NaN, "b": null}, + {"a": NaN, "b": NaN}, + {"a": null, "b": null}])"; + Check(schema, batch_input, options, expected); +} + TEST_F(TestSelectKWithRecordBatch, FullSelectKNullNaN) { auto schema = ::arrow::schema({ {field("a", float32())}, @@ -834,6 +978,51 @@ TEST_F(TestSelectKWithRecordBatch, FullSelectKNullNaN) { Check(schema, batch_input, options, expected); } +TEST_F(TestSelectKWithRecordBatch, FullSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + auto batch_input = R"([ + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN}, + {"a": NaN, "b": NaN}, + {"a": null, "b": NaN} + ])"; + std::vector sort_keys{ + SortKey("a", SortOrder::Ascending, NullPlacement::AtStart), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(10, sort_keys); + auto expected = R"([{"a": null, "b": NaN}, + {"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": NaN}, + {"a": NaN, "b": null} + ])"; + Check(schema, batch_input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}, + {"a": null, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"; + Check(schema, batch_input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtEnd; + expected = R"([{"a": NaN, "b": null}, + {"a": NaN, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}, + {"a": null, "b": NaN} + ])"; + Check(schema, batch_input, options, expected); +} + TEST_F(TestSelectKWithRecordBatch, BottomKOneColumnKey) { auto schema = ::arrow::schema({ {field("country", utf8())}, @@ -1147,26 +1336,26 @@ TEST_F(TestSelectKWithTable, FullSelectKNullNaN) { {"a": 1, "b": 3}, {"a": 3, "b": null}, {"a": 6, "b": NaN}, - {"a": 6, "b": null}, + {"a": 6, "b": null}, {"a": NaN, "b": 5}, - {"a": null, "b": 5}, + {"a": null, "b": 5}, {"a": null, "b": null}])"}; Check(schema, input, options, expected); options.sort_keys[0].null_placement = NullPlacement::AtStart; expected = {R"([ - {"a": null, "b": 5}, + {"a": null, "b": 5}, {"a": null, "b": null}, {"a": NaN, "b": 5}, {"a": 1, "b": 5}, {"a": 1, "b": 3}, {"a": 3, "b": null}, {"a": 6, "b": NaN}, - {"a": 6, "b": null} + {"a": 6, "b": null} ])"}; Check(schema, input, options, expected); options.sort_keys[1].null_placement = NullPlacement::AtStart; expected = {R"([ - {"a": null, "b": null}, + {"a": null, "b": null}, {"a": null, "b": 5}, {"a": NaN, "b": 5}, {"a": 1, "b": 5}, @@ -1178,5 +1367,96 @@ TEST_F(TestSelectKWithTable, FullSelectKNullNaN) { Check(schema, input, options, expected); } +TEST_F(TestSelectKWithTable, PartialSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + std::vector input = { + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN} + ])", + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"}; + + std::vector sort_keys{SortKey("a", SortOrder::Ascending), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(3, sort_keys); + + std::vector expected = { + R"([{"a": NaN, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN} + ])"}; + Check(schema, input, options, expected); +} + +TEST_F(TestSelectKWithTable, FullSelectKAllNullNaN) { + auto schema = ::arrow::schema({ + {field("a", float32())}, + {field("b", float64())}, + }); + std::vector input = { + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN} + ])", + R"([{"a": null, "b": null}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"}; + + std::vector sort_keys{SortKey("a", SortOrder::Ascending), + SortKey("b", SortOrder::Descending)}; + auto options = SelectKOptions(10, sort_keys); + + std::vector expected = { + R"([{"a": NaN, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null}, + {"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[0].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": NaN}, + {"a": null, "b": null}, + {"a": null, "b": null}, + {"a": NaN, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null} + ])"}; + Check(schema, input, options, expected); + options.sort_keys[1].null_placement = NullPlacement::AtStart; + expected = { + R"([{"a": null, "b": null}, + {"a": null, "b": null}, + {"a": null, "b": NaN}, + {"a": NaN, "b": null}, + {"a": NaN, "b": null}, + {"a": NaN, "b": NaN} + ])"}; + Check(schema, input, options, expected); +} + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/vector_select_k.cc b/cpp/src/arrow/compute/kernels/vector_select_k.cc index cc375919658c..ea072d179eee 100644 --- a/cpp/src/arrow/compute/kernels/vector_select_k.cc +++ b/cpp/src/arrow/compute/kernels/vector_select_k.cc @@ -17,7 +17,6 @@ #include #include -#include #include #include "arrow/compute/function.h" @@ -118,22 +117,22 @@ void HeapSortNonNullsToOutput(std::span non_null_input_range, Comparat return; } std::span heap = non_null_input_range.subspan(0, output_range.size()); - std::ranges::make_heap(heap, cmp); + std::make_heap(heap.begin(), heap.end(), cmp); std::span remaining_input = non_null_input_range.subspan(output_range.size()); for (uint64_t x_index : remaining_input) { if (cmp(x_index, heap.front())) { - std::ranges::pop_heap(heap, cmp); + std::pop_heap(heap.begin(), heap.end(), cmp); heap.back() = x_index; - std::ranges::push_heap(heap, cmp); + std::push_heap(heap.begin(), heap.end(), cmp); } } // fill output in reverse when destructing, // as the "worst" (next-to-would-have-been-replaced) element is at heap-top - for (auto& reverse_out_iter : std::ranges::reverse_view(output_range)) { - reverse_out_iter = heap.front(); // heap-top has the next element - std::ranges::pop_heap(heap, cmp); + for (int64_t i = output_range.size(); i > 0; --i) { + output_range[i - 1] = heap.front(); // heap-top has the next element + std::pop_heap(heap.begin(), heap.end(), cmp); // Decrease heap-size by one heap = heap.first(heap.size() - 1); } @@ -422,9 +421,8 @@ class ChunkedArraySelector : public TypeVisitor { // so the heap must have been completely filled DCHECK_EQ(heap.size(), output.non_null_like_range.size()); - for (uint64_t& reverse_out_iter : - std::ranges::reverse_view(output.non_null_like_range)) { - reverse_out_iter = + for (int64_t i = output.non_null_like_range.size(); i > 0; --i) { + output.non_null_like_range[i - 1] = heap.top().index + heap.top().offset; // heap-top has the next element heap.pop(); } diff --git a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc index 51152c64782e..8b40abfb67d6 100644 --- a/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc +++ b/cpp/src/arrow/flight/sql/odbc/odbc_impl/odbc_statement.cc @@ -784,11 +784,7 @@ SQLRETURN ODBCStatement::GetData(SQLSMALLINT record_number, SQLSMALLINT c_type, SQLRETURN ODBCStatement::GetMoreResults() { // Multiple result sets are not supported by Arrow protocol. - if (current_result_) { - return SQL_NO_DATA; - } else { - throw DriverException("Function sequence error", "HY010"); - } + return SQL_NO_DATA; } void ODBCStatement::GetColumnCount(SQLSMALLINT* column_count_ptr) { diff --git a/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc b/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc index 237626c27820..ba8b883aac47 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc +++ b/cpp/src/arrow/flight/sql/odbc/tests/statement_test.cc @@ -1984,14 +1984,7 @@ TYPED_TEST(StatementTest, TestSQLMoreResultsNoData) { } TYPED_TEST(StatementTest, TestSQLMoreResultsWithoutQuery) { -#ifdef __linux__ ASSERT_EQ(SQL_NO_DATA, SQLMoreResults(this->stmt)); -#else // Windows & Mac - // Verify function sequence error state is reported when SQLMoreResults is called - // without executing any queries - ASSERT_EQ(SQL_ERROR, SQLMoreResults(this->stmt)); - VerifyOdbcErrorState(SQL_HANDLE_STMT, this->stmt, kErrorStateHY010); -#endif } TYPED_TEST(StatementTest, TestSQLNativeSqlReturnsInputString) { diff --git a/cpp/src/arrow/util/bpacking.cc b/cpp/src/arrow/util/bpacking.cc index 1bf81df4f28f..b29927345691 100644 --- a/cpp/src/arrow/util/bpacking.cc +++ b/cpp/src/arrow/util/bpacking.cc @@ -32,9 +32,10 @@ struct UnpackDynamicFunction { static constexpr auto targets() { return std::array{ - ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar) // - ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon) // - ARROW_DISPATCH_TARGET_SVE128(&bpacking::unpack_sve128) // + ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar) // + ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon) // + // GH-50503: No SVE128 dispatch as it increases code size without + // increasing performance vs. Neon, and can produce ODR violations. ARROW_DISPATCH_TARGET_SVE256(&bpacking::unpack_sve256) // ARROW_DISPATCH_TARGET_SSE4_2(&bpacking::unpack_sse4_2) // ARROW_DISPATCH_TARGET_AVX2(&bpacking::unpack_avx2) // diff --git a/cpp/src/arrow/util/bpacking_benchmark.cc b/cpp/src/arrow/util/bpacking_benchmark.cc index 93d7cdf165c1..025b493c5927 100644 --- a/cpp/src/arrow/util/bpacking_benchmark.cc +++ b/cpp/src/arrow/util/bpacking_benchmark.cc @@ -206,11 +206,6 @@ BENCHMARK_UNPACK_ALL_TYPES_RUNTIME(Avx512Unaligned, false, bpacking::unpack_avx5 BENCHMARK_UNPACK_ALL_TYPES(NeonUnaligned, false, bpacking::unpack_neon); #endif -#if defined(ARROW_HAVE_RUNTIME_SVE128) -BENCHMARK_UNPACK_ALL_TYPES_RUNTIME(Sve128Unaligned, false, bpacking::unpack_sve128, - SVE128, "Sve128 not available"); -#endif - #if defined(ARROW_HAVE_RUNTIME_SVE256) BENCHMARK_UNPACK_ALL_TYPES_RUNTIME(Sve256Unaligned, false, bpacking::unpack_sve256, SVE256, "Sve256 not available"); diff --git a/cpp/src/arrow/util/bpacking_simd_128_alt.cc b/cpp/src/arrow/util/bpacking_simd_128_alt.cc deleted file mode 100644 index bd4799d3cd31..000000000000 --- a/cpp/src/arrow/util/bpacking_simd_128_alt.cc +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#if defined(ARROW_HAVE_RUNTIME_SVE128) -# define UNPACK_PLATFORM unpack_sve128 -# define KERNEL_PLATFORM KernelSve128 -#endif - -#if !defined(UNPACK_PLATFORM) -# error "This file must be compiled with a known SIMD micro architecture" -#endif - -#include - -#include "arrow/util/bpacking_dispatch_internal.h" -#include "arrow/util/bpacking_simd_internal.h" -#include "arrow/util/bpacking_simd_kernel_internal.h" - -namespace arrow::internal::bpacking { - -template -using KERNEL_PLATFORM = Kernel>; - -template -void UNPACK_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts) { - return unpack_jump(in, out, opts); -} - -template void UNPACK_PLATFORM(const uint8_t*, bool*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint8_t*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint16_t*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint32_t*, const UnpackOptions&); -template void UNPACK_PLATFORM(const uint8_t*, uint64_t*, const UnpackOptions&); - -} // namespace arrow::internal::bpacking - -#undef UNPACK_PLATFORM diff --git a/cpp/src/arrow/util/bpacking_simd_internal.h b/cpp/src/arrow/util/bpacking_simd_internal.h index d5a81baaec09..78aaa4a8f92a 100644 --- a/cpp/src/arrow/util/bpacking_simd_internal.h +++ b/cpp/src/arrow/util/bpacking_simd_internal.h @@ -53,34 +53,6 @@ extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128( #endif // UNPACK_ARCH128 #undef UNPACK_ARCH128 -#if defined(ARROW_HAVE_RUNTIME_SVE128) -# define UNPACK_ARCH128_ALT unpack_sve128 -#endif - -#if defined(UNPACK_ARCH128_ALT) - -template -ARROW_EXPORT void UNPACK_ARCH128_ALT(const uint8_t* in, Uint* out, - const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( // - const uint8_t* in, bool* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint8_t* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint16_t* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint32_t* out, const UnpackOptions& opts); - -extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128_ALT( - const uint8_t* in, uint64_t* out, const UnpackOptions& opts); - -#endif // UNPACK_ARCH128_ALT -#undef UNPACK_ARCH128_ALT - #if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256) # define UNPACK_ARCH256 unpack_sve256 #elif defined(UNPACK_ARCH256) || defined(ARROW_HAVE_RUNTIME_AVX2) diff --git a/cpp/src/arrow/util/bpacking_test.cc b/cpp/src/arrow/util/bpacking_test.cc index d4d588228e7e..0503a15110b6 100644 --- a/cpp/src/arrow/util/bpacking_test.cc +++ b/cpp/src/arrow/util/bpacking_test.cc @@ -301,15 +301,6 @@ TYPED_TEST(TestUnpack, UnpackAvx512) { TYPED_TEST(TestUnpack, UnpackNeon) { this->TestAll(&bpacking::unpack_neon); } #endif -#if defined(ARROW_HAVE_RUNTIME_SVE128) -TYPED_TEST(TestUnpack, UnpackSve128) { - if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE128)) { - GTEST_SKIP() << "Test requires SVE128"; - } - this->TestAll(&bpacking::unpack_sve128); -} -#endif - #if defined(ARROW_HAVE_RUNTIME_SVE256) TYPED_TEST(TestUnpack, UnpackSve256) { if (!CpuInfo::GetInstance()->IsSupported(CpuInfo::SVE256)) { diff --git a/cpp/src/arrow/util/rle_encoding_internal.h b/cpp/src/arrow/util/rle_encoding_internal.h index 825cd253df9b..5dc94f368d3c 100644 --- a/cpp/src/arrow/util/rle_encoding_internal.h +++ b/cpp/src/arrow/util/rle_encoding_internal.h @@ -811,7 +811,8 @@ auto RleBitPackedDecoder::GetBatch(value_type* out, } parser_.ParseWithCallable([&](auto run) { - using RunDecoder = RleBitPackedDecoderGetRunDecoder::type; + using RunDecoder = + typename RleBitPackedDecoderGetRunDecoder::type; ARROW_DCHECK_LT(values_read, batch_size); RunDecoder decoder(run, value_bit_width_); @@ -1124,7 +1125,8 @@ auto RleBitPackedDecoder::GetSpaced(Converter converter, } parser_.ParseWithCallable([&](auto run) { - using RunDecoder = RleBitPackedDecoderGetRunDecoder::type; + using RunDecoder = + typename RleBitPackedDecoderGetRunDecoder::type; RunDecoder decoder(run, value_bit_width_); @@ -1306,7 +1308,8 @@ auto RleBitPackedDecoder::GetBatchWithDict(const V* dictionary, } parser_.ParseWithCallable([&](auto run) { - using RunDecoder = RleBitPackedDecoderGetRunDecoder::type; + using RunDecoder = + typename RleBitPackedDecoderGetRunDecoder::type; RunDecoder decoder(run, value_bit_width_); diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index c6f4b01a717c..ff05b6bbae93 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -80,8 +80,8 @@ ARROW_JEMALLOC_BUILD_VERSION=5.3.0 ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa ARROW_LZ4_BUILD_VERSION=v1.10.0 ARROW_LZ4_BUILD_SHA256_CHECKSUM=537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b -ARROW_MIMALLOC_BUILD_VERSION=v3.3.1 -ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=42c16914168ac6741eeb407e83b93a12b2b7ee25a7e14e6b4807fab8b577a540 +ARROW_MIMALLOC_BUILD_VERSION=v3.4.1 +ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=37107a52c16baa80c5f74861dddda7b27bb9949e41a6637691867a94c88ca446 ARROW_NLOHMANN_JSON_BUILD_VERSION=v3.12.0 ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM=4b92eb0c06d10683f7447ce9406cb97cd4b453be18d7279320f7b2f025c10187 ARROW_OPENTELEMETRY_BUILD_VERSION=v1.21.0 diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 1773c216b5c0..5e76f60e55c1 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrow", - "version-string": "25.0.0-SNAPSHOT", + "version-string": "25.0.0", "dependencies": [ "abseil", { diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index 35e0d4d2c022..56c319fe7ef0 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -208,7 +208,7 @@ def verify_release_candidate(obj, base_branch, create_pr, for flag, group in zip(verify_flags, verify_groups): if flag: job_groups += f" --group {group}" - response.create_comment( + response.create_issue_comment( f"{command} {job_groups} --param " + f"release={version} --param rc={rc}") @@ -300,6 +300,10 @@ def asset_callback(task_name, task, asset): @crossbow.command() +@click.option('--base-branch', default='main', + help='Set base branch for the PR.') +@click.option('--head-branch', default=None, + help='Set head branch for the PR.') @click.option('--arrow-remote', '-r', default=None, help='Set GitHub remote explicitly, which is going to be cloned ' 'on the CI services. Note, that no validation happens ' @@ -313,7 +317,8 @@ def asset_callback(task_name, task, asset): @click.option('--pr-title', required=True, help='Track the job submitted on PR with given title') @click.pass_obj -def report_pr(obj, arrow_remote, crossbow, fetch, job_name, pr_title): +def report_pr(obj, base_branch, head_branch, arrow_remote, crossbow, + fetch, job_name, pr_title): arrow = obj['arrow'] queue = obj['queue'] if fetch: @@ -322,11 +327,12 @@ def report_pr(obj, arrow_remote, crossbow, fetch, job_name, pr_title): report = CommentReport(job, crossbow_repo=crossbow) target_arrow = Repo(path=arrow.path, remote_url=arrow_remote) - pull_request = target_arrow.github_pr(title=pr_title, + pull_request = target_arrow.github_pr(base=base_branch, head=head_branch, + title=pr_title, github_token=queue.github_token, create=False) # render the response comment's content on the PR - pull_request.create_comment(report.show()) + pull_request.create_issue_comment(report.show()) click.echo(f'Job is tracked on PR {pull_request.html_url}') diff --git a/dev/release/03-binary-submit.sh b/dev/release/03-binary-submit.sh index e3a0fc4ee7a1..acd0af5be49b 100755 --- a/dev/release/03-binary-submit.sh +++ b/dev/release/03-binary-submit.sh @@ -34,6 +34,7 @@ version_with_rc="${version}-rc${rc}" crossbow_job_prefix="release-${version_with_rc}" release_tag="apache-arrow-${version}-rc${rc}" rc_branch="release-${version_with_rc}" +maint_branch="maint-${version}" : ${ARROW_REPOSITORY:="apache/arrow"} : ${ARROW_BRANCH:=${release_tag}} @@ -57,5 +58,7 @@ job_name=$(archery crossbow latest-prefix --no-fetch ${crossbow_job_prefix}) archery crossbow report-pr \ --no-fetch \ --arrow-remote "https://github.com/${ARROW_REPOSITORY}" \ + --base-branch ${maint_branch} \ + --head-branch ${rc_branch} \ --job-name ${job_name} \ --pr-title "WIP: [Release] Verify ${rc_branch}" diff --git a/dev/release/verify-release-candidate-wheels.bat b/dev/release/verify-release-candidate-wheels.bat index df503f67d21a..431e96f24dd8 100644 --- a/dev/release/verify-release-candidate-wheels.bat +++ b/dev/release/verify-release-candidate-wheels.bat @@ -78,7 +78,7 @@ set PY_VERSION_NO_PERIOD=%PY_VERSION:.=% set CONDA_ENV_PATH=%_VERIFICATION_DIR%\_verify-wheel-%PY_VERSION% call conda create -p %CONDA_ENV_PATH% ^ - --no-shortcuts -f -q -y python=%PY_VERSION% ^ + --no-shortcuts -q -y python=%PY_VERSION% ^ || EXIT /B 1 call activate %CONDA_ENV_PATH% diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index ae49367ba6fa..fea869500972 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -362,7 +362,7 @@ install_conda() { maybe_setup_conda() { # Optionally setup conda environment with the passed dependencies local env="conda-${CONDA_ENV:-source}" - local pyver=${PYTHON_VERSION:-3} + local pyver=${PYTHON_VERSION:-3.12} if [ "${USE_CONDA}" -gt 0 ]; then show_info "Configuring Conda environment..." @@ -377,9 +377,10 @@ maybe_setup_conda() { if ! conda env list | cut -d" " -f 1 | grep $env; then mamba create -y -n $env python=${pyver} fi - # Install dependencies + # Install dependencies. Python version pinned so an unversioned + # python dependency does not replace it if [ $# -gt 0 ]; then - mamba install -y -n $env $@ + mamba install -y -n $env python=${pyver} $@ fi # Activate the environment conda activate $env diff --git a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb index de38408a5dc6..6062dc40b346 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb @@ -29,7 +29,7 @@ class ApacheArrowGlib < Formula desc "GLib bindings for Apache Arrow" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0-SNAPSHOT/apache-arrow-25.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0/apache-arrow-25.0.0.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index 00a0ef21939b..d5f8f5679270 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -29,7 +29,7 @@ class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0-SNAPSHOT/apache-arrow-25.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-25.0.0/apache-arrow-25.0.0.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog index e52cc96a2dbf..74414789b8c0 100644 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog @@ -1,3 +1,9 @@ +apache-arrow-apt-source (25.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Fri, 03 Jul 2026 07:48:10 -0000 + apache-arrow-apt-source (24.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in index 5cf507859e10..c243ebc5a75f 100644 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in +++ b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in @@ -85,6 +85,9 @@ else fi %changelog +* Fri Jul 03 2026 Raúl Cumplido - 25.0.0-1 +- New upstream release. + * Tue Apr 14 2026 Raúl Cumplido - 24.0.0-1 - New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/changelog b/dev/tasks/linux-packages/apache-arrow/debian/changelog index 5929ea8cca78..2dd4a3641253 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow/debian/changelog @@ -1,3 +1,9 @@ +apache-arrow (25.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Fri, 03 Jul 2026 07:48:10 -0000 + apache-arrow (24.0.0-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 97143a65d394..0eb529f3f171 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -886,6 +886,9 @@ Documentation for Apache Parquet GLib. %endif %changelog +* Fri Jul 03 2026 Raúl Cumplido - 25.0.0-1 +- New upstream release. + * Tue Apr 14 2026 Raúl Cumplido - 24.0.0-1 - New upstream release. diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index 435ea8ef9f04..c7e9aa3bf3de 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -1,15 +1,20 @@ [ { - "name": "25.0 (dev)", + "name": "26.0 (dev)", "version": "dev/", "url": "https://arrow.apache.org/docs/dev/" }, { - "name": "24.0 (stable)", + "name": "25.0 (stable)", "version": "", "url": "https://arrow.apache.org/docs/", "preferred": true }, + { + "name": "24.0", + "version": "24.0/", + "url": "https://arrow.apache.org/docs/24.0/" + }, { "name": "23.0", "version": "23.0/", diff --git a/docs/source/python/api/formats.rst b/docs/source/python/api/formats.rst index 0d2cd8975cde..57a5e824fab1 100644 --- a/docs/source/python/api/formats.rst +++ b/docs/source/python/api/formats.rst @@ -42,14 +42,11 @@ CSV Files .. _api.feather: -Feather Files (Deprecated) --------------------------- +Feather Files +------------- .. currentmodule:: pyarrow.feather -.. deprecated:: 24.0.0 - The Feather API is deprecated. Use the :ref:`IPC ` API instead. - .. autosummary:: :toctree: ../generated/ diff --git a/docs/source/python/feather.rst b/docs/source/python/feather.rst index 76520e912b67..33f8e76bf048 100644 --- a/docs/source/python/feather.rst +++ b/docs/source/python/feather.rst @@ -22,10 +22,6 @@ Feather File Format =================== -.. deprecated:: 24.0.0 - The ``pyarrow.feather`` module is deprecated. Feather V2 is the Arrow IPC - file format. Use :mod:`pyarrow.ipc` instead. See :ref:`ipc` for details. - Feather is a portable file format for storing Arrow tables or data frames (from languages like Python or R) that utilizes the :ref:`Arrow IPC format ` internally. Feather was created early in the Arrow project as a proof of @@ -39,8 +35,8 @@ R. There are two file format versions for Feather: * Version 1 (V1), a legacy version available starting in 2016, replaced by V2. V1 files are distinct from Arrow IPC files and lack many features, such as the ability to store all Arrow data types. V1 files also lack compression - support. We intend to maintain read support for V1 for the foreseeable - future. + support. Reading and writing V1 files is deprecated as of 25.0.0 and will + be removed in a future version. The ``pyarrow.feather`` module contains the read and write functions for the format. :func:`~pyarrow.feather.write_feather` accepts either a @@ -108,13 +104,23 @@ reduced disk IO requirements. Writing Version 1 (V1) Files ---------------------------- +.. deprecated:: 25.0.0 + Support for the legacy Feather V1 format is deprecated. Reading and + writing V1 files will be removed in a future version. Rewrite V1 files + in the Arrow IPC file format (Feather V2). + For compatibility with libraries without support for Version 2 files, you can -write the version 1 format by passing ``version=1`` to ``write_feather``. We -intend to maintain read support for V1 for the foreseeable future. +write the version 1 format by passing ``version=1`` to ``write_feather``. Migration to IPC ---------------- +.. note:: + + ``pyarrow.feather.write_feather`` and ``pyarrow.feather.read_table`` + equivalents will be provided in :mod:`pyarrow.ipc` before the + ``pyarrow.feather`` module is deprecated. + Since Feather V2 is the Arrow IPC file format, you can use the :mod:`pyarrow.ipc` module as a direct replacement: diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 5be5dcba39e0..47d56b9b262d 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -100,7 +100,7 @@ endfunction() set(CMAKE_CXX_STANDARD 20) -set(MLARROW_VERSION "25.0.0-SNAPSHOT") +set(MLARROW_VERSION "25.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MLARROW_BASE_VERSION "${MLARROW_VERSION}") project(mlarrow VERSION "${MLARROW_BASE_VERSION}") diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 1225a1140f3c..44f7854c3045 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -28,7 +28,7 @@ project(pyarrow) # which in turn meant that Py_GIL_DISABLED was not set. set(CMAKE_NO_SYSTEM_FROM_IMPORTED ON) -set(PYARROW_VERSION "25.0.0-SNAPSHOT") +set(PYARROW_VERSION "25.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}") # Generate SO version and full SO version diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5fc74969abfd..26bb6482a44a 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -1864,7 +1864,24 @@ cdef class Array(_PandasConvertible): lst : list """ self._assert_cpu() - return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + cdef int64_t i, n = self.length() + if maps_as_pydicts is not None: + # Converting maps to dicts has per-entry semantics (duplicate-key + # detection); use the Scalar-based conversion for exact behavior. + # TODO(GH-50429): this falls back to the Scalar path for the whole + # array even when the type contains no maps; threading + # maps_as_pydicts through _getitem_py keeps the fast paths instead. + return [x.as_py(maps_as_pydicts=maps_as_pydicts) for x in self] + # TODO(GH-50448): convert per range instead of per element to cut + # the per-element call overhead further. + return [self._getitem_py(i) for i in range(n)] + + cdef object _getitem_py(self, int64_t i): + # Return self[i] as a Python object, without creating a Python Scalar + # (nor, for nested types, per-row Array wrappers) where a subclass + # provides a specialization; this base implementation goes through + # Scalar.as_py and thus preserves its semantics exactly (see GH-50326). + return self.getitem(i).as_py() def tolist(self): """ @@ -2444,6 +2461,12 @@ cdef class BooleanArray(Array): """ Concrete class for Arrow arrays of boolean data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + return ( self.ap).Value(i) + @property def false_count(self): return ( self.ap).false_count() @@ -2458,6 +2481,34 @@ cdef class NumericArray(Array): A base class for Arrow numeric arrays. """ + cdef object _getitem_py(self, int64_t i): + cdef Type tid = self.ap.type_id() + if self.ap.IsNull(i): + return None + if tid == _Type_INT8: + return ( self.ap).Value(i) + elif tid == _Type_INT16: + return ( self.ap).Value(i) + elif tid == _Type_INT32: + return ( self.ap).Value(i) + elif tid == _Type_INT64: + return ( self.ap).Value(i) + elif tid == _Type_UINT8: + return ( self.ap).Value(i) + elif tid == _Type_UINT16: + return ( self.ap).Value(i) + elif tid == _Type_UINT32: + return ( self.ap).Value(i) + elif tid == _Type_UINT64: + return ( self.ap).Value(i) + elif tid == _Type_FLOAT: + return ( self.ap).Value(i) + elif tid == _Type_DOUBLE: + return ( self.ap).Value(i) + # Subclasses whose as_py returns non-primitive objects (dates, times, + # timestamps, durations, half floats, ...) use the exact Scalar path. + return Array._getitem_py(self, i) + cdef class IntegerArray(NumericArray): """ @@ -2776,6 +2827,16 @@ cdef class ListArray(BaseListArray): Concrete class for Arrow arrays of a list data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -2961,6 +3022,16 @@ cdef class LargeListArray(BaseListArray): Identical to ListArray, but 64-bit offsets. """ + cdef object _getitem_py(self, int64_t i): + cdef CLargeListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(offsets, values, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3551,6 +3622,19 @@ cdef class MapArray(ListArray): Concrete class for Arrow arrays of a map data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = (self.keys, self.items) + cdef Array keys = ( self._children_cache)[0] + cdef Array items = ( self._children_cache)[1] + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + # Matches MapScalar.as_py with the default maps_as_pydicts=None: + # an association list of (key, value) tuples. + return [(keys._getitem_py(j), items._getitem_py(j)) for j in range(start, end)] + @staticmethod def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None): """ @@ -3688,6 +3772,16 @@ cdef class FixedSizeListArray(BaseListArray): Concrete class for Arrow arrays of a fixed size list data type. """ + cdef object _getitem_py(self, int64_t i): + cdef CFixedSizeListArray* arr = self.ap + if arr.IsNull(i): + return None + if self._children_cache is None: + self._children_cache = pyarrow_wrap_array(arr.values()) + cdef Array values = self._children_cache + cdef int64_t j, start = arr.value_offset(i), end = arr.value_offset(i + 1) + return [values._getitem_py(j) for j in range(start, end)] + @staticmethod def from_arrays(values, list_size=None, DataType type=None, mask=None): """ @@ -3974,6 +4068,13 @@ cdef class StringArray(Array): Concrete class for Arrow arrays of string (or utf8) data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + # Matches StringScalar.as_py, which is str(buf, 'utf8'). + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, @@ -4006,6 +4107,12 @@ cdef class LargeStringArray(Array): Concrete class for Arrow arrays of large string (or utf8) data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) + @staticmethod def from_buffers(int length, Buffer value_offsets, Buffer data, Buffer null_bitmap=None, int null_count=-1, @@ -4038,11 +4145,24 @@ cdef class StringViewArray(Array): Concrete class for Arrow arrays of string (or utf8) view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyUnicode_DecodeUTF8(view.data(), view.size(), NULL) + cdef class BinaryArray(Array): """ Concrete class for Arrow arrays of variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) + @property def total_values_length(self): """ @@ -4056,6 +4176,13 @@ cdef class LargeBinaryArray(Array): """ Concrete class for Arrow arrays of large variable-sized binary data type. """ + + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) + @property def total_values_length(self): """ @@ -4070,6 +4197,12 @@ cdef class BinaryViewArray(Array): Concrete class for Arrow arrays of variable-sized binary view data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef cpp_string_view view = ( self.ap).GetView(i) + return cp.PyBytes_FromStringAndSize(view.data(), view.size()) + cdef class DictionaryArray(Array): """ @@ -4229,6 +4362,28 @@ cdef class StructArray(Array): Concrete class for Arrow arrays of a struct data type. """ + cdef object _getitem_py(self, int64_t i): + if self.ap.IsNull(i): + return None + cdef int64_t k, num_fields = self.type.num_fields + if self._children_cache is None: + names = [self.type.field(k).name for k in range(num_fields)] + if len(set(names)) != len(names): + # Matches StructScalar.as_py + raise ValueError( + "Converting to Python dictionary is not supported when " + "duplicate field names are present") + self._children_cache = ( + names, [self.field(k) for k in range(num_fields)]) + names = ( self._children_cache)[0] + fields = ( self._children_cache)[1] + cdef Array field_arr + result = {} + for k in range(num_fields): + field_arr = fields[k] + result[names[k]] = field_arr._getitem_py(i) + return result + def field(self, index): """ Retrieves the child array belonging to field. diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py index 68f708c91489..60d59b0e0b66 100644 --- a/python/pyarrow/feather.py +++ b/python/pyarrow/feather.py @@ -32,9 +32,6 @@ class FeatherDataset: """ Encapsulates details of reading a list of Feather files. - .. deprecated:: 24.0.0 - Use :func:`pyarrow.dataset.dataset` with ``format='ipc'`` instead. - Parameters ---------- path_or_paths : List[str] @@ -44,12 +41,6 @@ class FeatherDataset: """ def __init__(self, path_or_paths, validate_schema=True): - warnings.warn( - "pyarrow.feather.FeatherDataset is deprecated as of 24.0.0. " - "Use pyarrow.dataset.dataset() with format='ipc' instead.", - FutureWarning, - stacklevel=2 - ) self.paths = path_or_paths self.validate_schema = validate_schema @@ -127,11 +118,6 @@ def write_feather(df, dest, compression=None, compression_level=None, """ Write a pandas.DataFrame to Feather format. - .. deprecated:: 24.0.0 - Use :func:`pyarrow.ipc.new_file` / - :class:`pyarrow.ipc.RecordBatchFileWriter` instead. - Feather V2 is the Arrow IPC file format. - Parameters ---------- df : pandas.DataFrame or pyarrow.Table @@ -150,15 +136,20 @@ def write_feather(df, dest, compression=None, compression_level=None, which is currently 64K version : int, default 2 Feather file version. Version 2 is the current. Version 1 is the more - limited legacy format + limited legacy format. + + .. deprecated:: 25.0.0 + Writing Feather V1 files is deprecated. Use the default + ``version=2`` to write Arrow IPC files instead. """ - warnings.warn( - "pyarrow.feather.write_feather is deprecated as of 24.0.0. " - "Use pyarrow.ipc.new_file() / RecordBatchFileWriter instead. " - "Feather V2 is the Arrow IPC file format.", - FutureWarning, - stacklevel=2 - ) + if version == 1: + warnings.warn( + "Feather V1 files are deprecated as of 25.0.0 and support will " + "be removed in a future version. Use the default version=2 to " + "write Arrow IPC files instead.", + DeprecationWarning, + stacklevel=2 + ) if _pandas_api.have_pandas: if (_pandas_api.has_sparse and isinstance(df, _pandas_api.pd.SparseDataFrame)): @@ -223,11 +214,6 @@ def read_feather(source, columns=None, use_threads=True, Read a pandas.DataFrame from Feather format. To read as pyarrow.Table use feather.read_table. - .. deprecated:: 24.0.0 - Use :func:`pyarrow.ipc.open_file` / - :class:`pyarrow.ipc.RecordBatchFileReader` instead. - Feather V2 is the Arrow IPC file format. - Parameters ---------- source : str file path, or file-like object @@ -249,13 +235,6 @@ def read_feather(source, columns=None, use_threads=True, df : pandas.DataFrame The contents of the Feather file as a pandas.DataFrame """ - warnings.warn( - "pyarrow.feather.read_feather is deprecated as of 24.0.0. " - "Use pyarrow.ipc.open_file() / RecordBatchFileReader instead. " - "Feather V2 is the Arrow IPC file format.", - FutureWarning, - stacklevel=2 - ) return (_read_table_internal( source, columns=columns, memory_map=memory_map, use_threads=use_threads).to_pandas(use_threads=use_threads, **kwargs)) @@ -265,11 +244,20 @@ def _read_table_internal(source, columns=None, memory_map=False, use_threads=True): """ Internal implementation for reading a Feather file as a pyarrow.Table. - Does not emit deprecation warnings. + Emits a deprecation warning if the file is a legacy Feather V1 file. """ reader = _feather.FeatherReader( source, use_memory_map=memory_map, use_threads=use_threads) + if reader.version < 3: + warnings.warn( + "Feather V1 files are deprecated as of 25.0.0 and support will " + "be removed in a future version. Consider rewriting this file " + "in the Arrow IPC file format (Feather V2).", + DeprecationWarning, + stacklevel=3 + ) + if columns is None: return reader.read() @@ -302,11 +290,6 @@ def read_table(source, columns=None, memory_map=False, use_threads=True): """ Read a pyarrow.Table from Feather format - .. deprecated:: 24.0.0 - Use :func:`pyarrow.ipc.open_file` / - :class:`pyarrow.ipc.RecordBatchFileReader` instead. - Feather V2 is the Arrow IPC file format. - Parameters ---------- source : str file path, or file-like object @@ -324,13 +307,6 @@ def read_table(source, columns=None, memory_map=False, use_threads=True): table : pyarrow.Table The contents of the Feather file as a pyarrow.Table """ - warnings.warn( - "pyarrow.feather.read_table is deprecated as of 24.0.0. " - "Use pyarrow.ipc.open_file() / RecordBatchFileReader instead. " - "Feather V2 is the Arrow IPC file format.", - FutureWarning, - stacklevel=2 - ) return _read_table_internal(source, columns=columns, memory_map=memory_map, use_threads=use_threads) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 8b4786ecbf13..e57c6d0d92d4 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -264,7 +264,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string Diff(const CArray& other) c_bool Equals(const CArray& arr) - c_bool IsNull(int i) + c_bool IsNull(int64_t i) shared_ptr[CArrayData] data() @@ -675,87 +675,87 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string* result) cdef cppclass CBooleanArray" arrow::BooleanArray"(CArray): - c_bool Value(int i) + c_bool Value(int64_t i) int64_t false_count() int64_t true_count() cdef cppclass CUInt8Array" arrow::UInt8Array"(CArray): - uint8_t Value(int i) + uint8_t Value(int64_t i) cdef cppclass CInt8Array" arrow::Int8Array"(CArray): - int8_t Value(int i) + int8_t Value(int64_t i) cdef cppclass CUInt16Array" arrow::UInt16Array"(CArray): - uint16_t Value(int i) + uint16_t Value(int64_t i) cdef cppclass CInt16Array" arrow::Int16Array"(CArray): - int16_t Value(int i) + int16_t Value(int64_t i) cdef cppclass CUInt32Array" arrow::UInt32Array"(CArray): - uint32_t Value(int i) + uint32_t Value(int64_t i) cdef cppclass CInt32Array" arrow::Int32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CUInt64Array" arrow::UInt64Array"(CArray): - uint64_t Value(int i) + uint64_t Value(int64_t i) cdef cppclass CInt64Array" arrow::Int64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CDate32Array" arrow::Date32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CDate64Array" arrow::Date64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CTime32Array" arrow::Time32Array"(CArray): - int32_t Value(int i) + int32_t Value(int64_t i) cdef cppclass CTime64Array" arrow::Time64Array"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CTimestampArray" arrow::TimestampArray"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CDurationArray" arrow::DurationArray"(CArray): - int64_t Value(int i) + int64_t Value(int64_t i) cdef cppclass CMonthDayNanoIntervalArray \ "arrow::MonthDayNanoIntervalArray"(CArray): pass cdef cppclass CHalfFloatArray" arrow::HalfFloatArray"(CArray): - uint16_t Value(int i) + uint16_t Value(int64_t i) cdef cppclass CFloatArray" arrow::FloatArray"(CArray): - float Value(int i) + float Value(int64_t i) cdef cppclass CDoubleArray" arrow::DoubleArray"(CArray): - double Value(int i) + double Value(int64_t i) cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray): - const uint8_t* GetValue(int i) + const uint8_t* GetValue(int64_t i) cdef cppclass CDecimal32Array" arrow::Decimal32Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal64Array" arrow::Decimal64Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal128Array" arrow::Decimal128Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CDecimal256Array" arrow::Decimal256Array"( CFixedSizeBinaryArray ): - c_string FormatValue(int i) + c_string FormatValue(int64_t i) cdef cppclass CListArray" arrow::ListArray"(CArray): @staticmethod @@ -776,8 +776,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: ) const int32_t* raw_value_offsets() - int32_t value_offset(int i) - int32_t value_length(int i) + int32_t value_offset(int64_t i) + int32_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CDataType] value_type() @@ -800,8 +800,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CBuffer] null_bitmap ) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CDataType] value_type() @@ -819,8 +819,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CDataType], shared_ptr[CBuffer] null_bitmap) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CDataType] value_type() @@ -850,8 +850,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const int32_t* raw_value_offsets() const int32_t* raw_value_sizes() - int32_t value_offset(int i) - int32_t value_length(int i) + int32_t value_offset(int64_t i) + int32_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CArray] sizes() @@ -881,8 +881,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CMemoryPool* pool ) - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CArray] offsets() shared_ptr[CArray] sizes() @@ -911,8 +911,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CArray] keys() shared_ptr[CArray] items() CMapType* map_type() - int64_t value_offset(int i) - int64_t value_length(int i) + int64_t value_offset(int64_t i) + int64_t value_length(int64_t i) shared_ptr[CArray] values() shared_ptr[CDataType] value_type() @@ -941,18 +941,20 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const vector[c_string]& field_names, const vector[int8_t]& type_codes) - int32_t value_offset(int i) + int32_t value_offset(int64_t i) shared_ptr[CBuffer] value_offsets() cdef cppclass CBinaryArray" arrow::BinaryArray"(CArray): - const uint8_t* GetValue(int i, int32_t* length) + const uint8_t* GetValue(int64_t i, int32_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int32_t value_offset(int64_t i) int32_t value_length(int64_t i) int32_t total_values_length() cdef cppclass CLargeBinaryArray" arrow::LargeBinaryArray"(CArray): - const uint8_t* GetValue(int i, int64_t* length) + const uint8_t* GetValue(int64_t i, int64_t* length) + cpp_string_view GetView(int64_t i) shared_ptr[CBuffer] value_data() int64_t value_offset(int64_t i) int64_t value_length(int64_t i) @@ -965,7 +967,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int64_t null_count, int64_t offset) - c_string GetString(int i) + c_string GetString(int64_t i) cdef cppclass CLargeStringArray" arrow::LargeStringArray" \ (CLargeBinaryArray): @@ -975,7 +977,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int64_t null_count, int64_t offset) - c_string GetString(int i) + c_string GetString(int64_t i) + + cdef cppclass CBinaryViewArray" arrow::BinaryViewArray"(CArray): + cpp_string_view GetView(int64_t i) + + cdef cppclass CStringViewArray" arrow::StringViewArray"(CBinaryViewArray): + pass cdef cppclass CStructArray" arrow::StructArray"(CArray): CStructArray(shared_ptr[CDataType]& type, int64_t length, diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 683faa7855c5..38f1ac69a807 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -288,8 +288,15 @@ cdef class Array(_PandasConvertible): # To allow Table to propagate metadata to pandas.Series object _name + cdef: + # Lazily wrapped child array(s) reused by _getitem_py (see GH-50326). + # Appended after the pre-existing attributes to keep their offsets + # stable for extensions compiled against an older pyarrow. + object _children_cache + cdef void init(self, const shared_ptr[CArray]& sp_array) except * cdef getitem(self, int64_t i) + cdef object _getitem_py(self, int64_t i) cdef int64_t length(self) cdef void _assert_cpu(self) except * diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index adc3e097b54a..c1e3f0128be8 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -465,6 +465,64 @@ def test_array_getitem_numpy_scalars(): assert arr[np.int32(idx)].as_py() == lst[idx] +def test_to_pylist_bulk_paths(): + # GH-50326: to_pylist converts through scalar-free _getitem_py + # specializations; the result must match the per-scalar conversion + # exactly. + arrays = [ + pa.array([[1, None, 3], None, [], [4]], type=pa.list_(pa.int32())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.list_(pa.string())), + pa.array([["a", None], None, [], ["bcd", ""]], + type=pa.large_list(pa.large_string())), + pa.array([[1, None], None, [3, 4]], type=pa.list_(pa.int32(), 2)), + pa.array([[[1], [2, None]], None, [None, [3]]], + type=pa.list_(pa.list_(pa.int32()))), + pa.array([[("k1", 1), ("k2", None)], None, []], + type=pa.map_(pa.string(), pa.int32())), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.string()), + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}"], + type=pa.large_string()), + pa.array([b"a\x00b", None, b"", b"\xff"], type=pa.binary()), + pa.array([b"a\x00b", None, b""], type=pa.large_binary()), + # View types store short values inline and long values out-of-line; + # cover both, plus NUL bytes and non-ASCII data. + pa.array(["a", None, "", "\N{GRINNING FACE} \N{SNOWMAN}", + "long string exceeding the inline view size"], + type=pa.string_view()), + pa.array([b"a\x00b", None, b"", b"\xff", + b"long binary value exceeding the inline view size"], + type=pa.binary_view()), + pa.array([[b"x", None, b"\x00y"], None, []], + type=pa.list_(pa.binary())), + pa.array([1, None, -(2**62), 2**62], type=pa.int64()), + pa.array([0, None, 2**63 + 7], type=pa.uint64()), + pa.array([-128, 127, None], type=pa.int8()), + pa.array([1.5, None, -0.5], type=pa.float64()), + pa.array([1.5, None], type=pa.float32()), + pa.array([True, None, False], type=pa.bool_()), + pa.array([{"a": 1, "b": "x"}, None, {"a": None, "b": None}], + type=pa.struct([("a", pa.int32()), ("b", pa.string())])), + pa.array([], type=pa.list_(pa.int32())), + pa.array([None, None], type=pa.list_(pa.string())), + ] + for arr in arrays: + for view in (arr, arr.slice(1), arr.slice(0, 2), arr.slice(2)): + assert view.to_pylist() == [x.as_py() for x in view] + + # Values inside numeric lists must stay Python ints/None, never floats + result = pa.array([[1, None, 3]], type=pa.list_(pa.int32())).to_pylist() + assert result == [[1, None, 3]] + assert [type(x) for x in result[0]] == [int, type(None), int] + + # Duplicate struct field names raise like StructScalar.as_py does + dup = pa.StructArray.from_arrays( + [pa.array([1, 2]), pa.array(["a", "b"])], names=["x", "x"]) + with pytest.raises(ValueError, match="duplicate field names"): + dup.to_pylist() + + def test_array_slice(): arr = pa.array(range(10)) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 63c537eae5fc..74693317d711 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -1927,7 +1927,6 @@ def test_fragments_parquet_subset_with_nested_fields(tempdir): @pytest.mark.pandas @pytest.mark.parquet -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_fragments_repr(tempdir, dataset): # partitioned parquet dataset fragment = list(dataset.get_fragments())[0] @@ -3700,7 +3699,7 @@ def test_column_names_encoding(tempdir, dataset_reader): assert dataset_transcoded.to_table().equals(expected_table) -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") +@pytest.mark.filterwarnings("ignore:Feather V1:DeprecationWarning") def test_feather_format(tempdir, dataset_reader): from pyarrow.feather import write_feather @@ -4082,7 +4081,6 @@ def test_dataset_project_null_column(tempdir, dataset_reader): assert dataset_reader.to_table(dataset).equals(expected) -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_dataset_project_columns(tempdir, dataset_reader): # basic column re-projection with expressions from pyarrow import feather @@ -4434,7 +4432,6 @@ def test_write_dataset_with_dataset(tempdir): @pytest.mark.pandas -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_write_dataset_existing_data(tempdir): directory = tempdir / 'ds' table = pa.table({'b': ['x', 'y', 'z'], 'c': [1, 2, 3]}) @@ -5058,7 +5055,6 @@ def test_write_dataset_arrow_schema_metadata(tempdir): assert result["a"].type.tz == "Europe/Brussels" -@pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") def test_write_dataset_schema_metadata(tempdir): # ensure that schema metadata gets written from pyarrow import feather diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py index c04cef534ee3..8c9e7eb437e6 100644 --- a/python/pyarrow/tests/test_feather.py +++ b/python/pyarrow/tests/test_feather.py @@ -41,10 +41,10 @@ except ImportError: pass -# Suppress deprecation warnings for existing tests since pyarrow.feather -# is deprecated as of 24.0.0 +# Suppress deprecation warnings for existing tests that intentionally +# exercise the deprecated Feather V1 format pytestmark = pytest.mark.filterwarnings( - "ignore:pyarrow.feather:FutureWarning" + "ignore:Feather V1:DeprecationWarning" ) @@ -894,39 +894,44 @@ def test_feather_datetime_resolution_arrow_to_pandas(tempdir): # --- Deprecation warning tests --- @pytest.mark.pandas -@pytest.mark.filterwarnings("default:pyarrow.feather:FutureWarning") -def test_feather_deprecation_warnings(tempdir): +@pytest.mark.filterwarnings("default:Feather V1:DeprecationWarning") +def test_feather_v1_deprecation_warnings(tempdir): table = pa.table({"a": [1, 2, 3]}) path = str(tempdir / "test.feather") - with pytest.warns(FutureWarning, match="write_feather is deprecated"): - write_feather(table, path) + with pytest.warns(DeprecationWarning, match="Feather V1"): + write_feather(table, path, version=1) - with pytest.warns(FutureWarning, match="read_table is deprecated"): + with pytest.warns(DeprecationWarning, match="Feather V1"): read_table(path) - with pytest.warns(FutureWarning, match="read_feather is deprecated"): + with pytest.warns(DeprecationWarning, match="Feather V1"): read_feather(path) -@pytest.mark.filterwarnings("default:pyarrow.feather:FutureWarning") -def test_feather_dataset_deprecated(): - with pytest.warns(FutureWarning, match="FeatherDataset is deprecated"): - FeatherDataset([]) +def test_feather_v2_no_deprecation_warning(tempdir): + table = pa.table({"a": [1, 2, 3]}) + path = str(tempdir / "test.feather") + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + write_feather(table, path) + read_table(path) + FeatherDataset([path]).read_table() @pytest.mark.pandas -@pytest.mark.filterwarnings("default:pyarrow.feather:FutureWarning") -def test_read_feather_no_double_warning(tempdir): - """Verify read_feather emits exactly one FutureWarning, not two.""" +@pytest.mark.filterwarnings("default:Feather V1:DeprecationWarning") +def test_read_feather_v1_no_double_warning(tempdir): + """Verify reading a V1 file emits one DeprecationWarning, not two.""" table = pa.table({"a": [1, 2, 3]}) path = str(tempdir / "test.feather") with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - write_feather(table, path) + warnings.simplefilter("ignore", DeprecationWarning) + write_feather(table, path, version=1) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") read_feather(path) - future_warnings = [x for x in w if issubclass(x.category, - FutureWarning)] - assert len(future_warnings) == 1 + v1_warnings = [x for x in w if issubclass(x.category, + DeprecationWarning)] + assert len(v1_warnings) == 1 diff --git a/python/pyproject.toml b/python/pyproject.toml index 0054e1a26828..b18f74e3e12c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -107,7 +107,7 @@ root = '..' version_file = 'pyarrow/_generated_version.py' version_scheme = 'guess-next-dev' git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' -fallback_version = '25.0.0a0' +fallback_version = '25.0.0' # TODO: Enable type checking once stubs are merged [tool.mypy] diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 99a1ff318845..0d2a743dd00b 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -1,6 +1,6 @@ Package: arrow Title: Integration to 'Apache' 'Arrow' -Version: 24.0.0.9000 +Version: 25.0.0 Authors@R: c( person("Neal", "Richardson", email = "neal.p.richardson@gmail.com", role = c("aut")), person("Ian", "Cook", email = "ianmcook@gmail.com", role = c("aut")), diff --git a/r/NEWS.md b/r/NEWS.md index b80639fd0e3d..f90d94d1bacb 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -17,7 +17,7 @@ under the License. --> -# arrow 24.0.0.9000 +# arrow 25.0.0 # arrow 24.0.0 diff --git a/r/pkgdown/assets/versions.html b/r/pkgdown/assets/versions.html index b66357ec165a..69b1e75fedfd 100644 --- a/r/pkgdown/assets/versions.html +++ b/r/pkgdown/assets/versions.html @@ -1,7 +1,8 @@ -

24.0.0.9000 (dev)

-

24.0.0 (release)

+

25.0.0.9000 (dev)

+

25.0.0 (release)

+

24.0.0

23.0.1

22.0.0

21.0.0

diff --git a/r/pkgdown/assets/versions.json b/r/pkgdown/assets/versions.json index 2fd5f6b05cdc..29f95e742b35 100644 --- a/r/pkgdown/assets/versions.json +++ b/r/pkgdown/assets/versions.json @@ -1,12 +1,16 @@ [ { - "name": "24.0.0.9000 (dev)", + "name": "25.0.0.9000 (dev)", "version": "dev/" }, { - "name": "24.0.0 (release)", + "name": "25.0.0 (release)", "version": "" }, + { + "name": "24.0.0", + "version": "24.0/" + }, { "name": "23.0.1", "version": "23.0/" diff --git a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb index 62cd8d9e40eb..5207d09980e9 100644 --- a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb +++ b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowCUDA - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb index 4b255e7b12ad..4b4e0d77adc6 100644 --- a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb +++ b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowDataset - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb index babf047e67b5..e230978541d2 100644 --- a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb +++ b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlightSQL - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight/lib/arrow-flight/version.rb b/ruby/red-arrow-flight/lib/arrow-flight/version.rb index c3e0a064f04e..886d0b451f22 100644 --- a/ruby/red-arrow-flight/lib/arrow-flight/version.rb +++ b/ruby/red-arrow-flight/lib/arrow-flight/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlight - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-format/lib/arrow-format/version.rb b/ruby/red-arrow-format/lib/arrow-format/version.rb index f61c89f1fe34..1e319d1ffd73 100644 --- a/ruby/red-arrow-format/lib/arrow-format/version.rb +++ b/ruby/red-arrow-format/lib/arrow-format/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFormat - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow/lib/arrow/version.rb b/ruby/red-arrow/lib/arrow/version.rb index b28b7aa57cf4..719317f1ef09 100644 --- a/ruby/red-arrow/lib/arrow/version.rb +++ b/ruby/red-arrow/lib/arrow/version.rb @@ -16,7 +16,7 @@ # under the License. module Arrow - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-gandiva/lib/gandiva/version.rb b/ruby/red-gandiva/lib/gandiva/version.rb index dd2dad6e0555..8f4da7bb04f6 100644 --- a/ruby/red-gandiva/lib/gandiva/version.rb +++ b/ruby/red-gandiva/lib/gandiva/version.rb @@ -16,7 +16,7 @@ # under the License. module Gandiva - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-parquet/lib/parquet/version.rb b/ruby/red-parquet/lib/parquet/version.rb index 8c23e7f66b86..75393c016831 100644 --- a/ruby/red-parquet/lib/parquet/version.rb +++ b/ruby/red-parquet/lib/parquet/version.rb @@ -16,7 +16,7 @@ # under the License. module Parquet - VERSION = "25.0.0-SNAPSHOT" + VERSION = "25.0.0" module Version numbers, TAG = VERSION.split("-")