From 688cfcd24af3bb33ae0a55c466b0f41062f01353 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Fri, 9 Oct 2026 10:06:03 -0500 Subject: [PATCH 01/15] Add adaptive compression for Parquet V2 pages Add min_space_savings to control how much space compression must save. V2 pages below the threshold are written uncompressed. Keep the default behavior and V1 pages unchanged. --- cpp/src/parquet/column_writer.cc | 13 ++++++- cpp/src/parquet/column_writer_test.cc | 55 ++++++++++++++++++++++++++- cpp/src/parquet/properties.h | 24 +++++++++++- cpp/src/parquet/properties_test.cc | 27 +++++++++++++ 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 3296af62f0c4..9d8026c3cc08 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -1057,8 +1057,17 @@ void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size, bool page_is_compressed = false; if (pager_->has_compressor() && values->size() > 0) { pager_->Compress(*values, compressor_temp_buffer_.get()); - if (compressor_temp_buffer_->size() < values->size()) { - page_is_compressed = true; + + if (const auto minimum = properties_->min_space_savings()) { + + //Checks if compressed size meets the minimum threshold + const double savings = + 1.0 - static_cast(compressor_temp_buffer_->size()) / values->size(); + page_is_compressed = savings >= *minimum; + } else { + // Keeps original behavior + // Allows users keep min_space_savings unset + page_is_compressed = compressor_temp_buffer_->size() < values->size(); } } std::shared_ptr compressed_values = diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index 69ec07c1b78f..a65facf4e8a4 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include #include +#include #include #include @@ -112,7 +114,8 @@ class TestPrimitiveWriter : public PrimitiveTypedTest { const ParquetVersion::type version = ParquetVersion::PARQUET_1_0, const ParquetDataPageVersion data_page_version = ParquetDataPageVersion::V1, bool enable_checksum = false, int64_t page_size = kDefaultDataPageSize, - int64_t max_rows_per_page = kDefaultMaxRowsPerPage) { + int64_t max_rows_per_page = kDefaultMaxRowsPerPage, + std::optional min_space_savings = {}) { sink_ = CreateOutputStream(); WriterProperties::Builder wp_builder; wp_builder.version(version)->data_page_version(data_page_version); @@ -130,6 +133,7 @@ class TestPrimitiveWriter : public PrimitiveTypedTest { wp_builder.max_statistics_size(column_properties.max_statistics_size()); wp_builder.data_pagesize(page_size); wp_builder.max_rows_per_page(max_rows_per_page); + wp_builder.min_space_savings(min_space_savings); writer_properties_ = wp_builder.build(); metadata_ = ColumnChunkMetaDataBuilder::Make(writer_properties_, this->descr_); @@ -2084,6 +2088,55 @@ TEST_F(TestValuesWriterInt32Type, AvoidCompressedInDataPageV2) { verify_only_one_uncompressed_page(/*total_num_values=*/1); } } + +// Tests if compression works when: +// - min_space_savings is unset +// - min_space_savings = 0.1 +// - min_space_savings = 1 +TEST_F(TestValuesWriterInt32Type, MinSpaceSavingsDataPageV2) { + + // Used zeros because it compresses well + this->SetUpSchema(Repetition::OPTIONAL); + this->descr_ = this->schema_.Column(0); + this->GenerateData(SMALL_SIZE); + std::fill(this->values_.begin(), this->values_.end(), 0); + + ColumnProperties column_properties; + column_properties.set_compression(Compression::ZSTD); + + const std::vector, bool>> cases = { + {std::nullopt, true}, + {0.1, true}, + {1.0, false}, + }; + + for (const auto& [minimum, expected_compressed] : cases) { + auto writer = this->BuildWriter( + SMALL_SIZE, column_properties, ParquetVersion::PARQUET_2_LATEST, + ParquetDataPageVersion::V2, false, kDefaultDataPageSize, + kDefaultMaxRowsPerPage, minimum); + + writer->WriteBatch(SMALL_SIZE, this->def_levels_.data(), nullptr, + this->values_ptr_); + writer->Close(); + + ASSERT_OK_AND_ASSIGN(auto buffer, this->sink_->Finish()); + + auto page_reader = PageReader::Open( + std::make_shared<::arrow::io::BufferReader>(buffer), SMALL_SIZE, + Compression::ZSTD, default_reader_properties(), *this->descr_); + + auto page = page_reader->NextPage(); + + ASSERT_NE(page, nullptr); + ASSERT_EQ(PageType::DATA_PAGE_V2, page->type()); + + auto data_page = std::static_pointer_cast(page); + + ASSERT_EQ(expected_compressed, data_page->is_compressed()); + } +} + #endif // Test writing and reading geometry columns diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index 89f2be42bffd..3bb721c7b560 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -403,6 +403,7 @@ class PARQUET_EXPORT WriterProperties { max_rows_per_page_(properties.max_rows_per_page()), version_(properties.version()), data_page_version_(properties.data_page_version()), + min_space_savings_(properties.min_space_savings()), created_by_(properties.created_by()), store_decimal_as_integer_(properties.store_decimal_as_integer()), page_checksum_enabled_(properties.page_checksum_enabled()), @@ -529,6 +530,13 @@ class PARQUET_EXPORT WriterProperties { return this; } + /// Used to set minimum threshold for V2 page data values compression + /// If min_space_savings unset, keep compression if compressed value is smaller. + Builder* min_space_savings(std::optional min_space_savings) { + min_space_savings_ = min_space_savings; + return this; + } + /// Specify the Parquet file version. /// Default PARQUET_2_6. Builder* version(ParquetVersion::type version) { @@ -883,6 +891,11 @@ class PARQUET_EXPORT WriterProperties { /// \brief Build the WriterProperties with the builder parameters. /// \return The WriterProperties defined by the builder. std::shared_ptr build() { + // checks for invalid input of min_space_savings + if (min_space_savings_ && + !(*min_space_savings_ >= 0.0 && *min_space_savings_ <= 1.0)) { + throw ParquetException("min_space_savings must be in range [0, 1]"); + } std::unordered_map column_properties; auto get = [&](const std::string& key) -> ColumnProperties& { auto it = column_properties.find(key); @@ -918,7 +931,7 @@ class PARQUET_EXPORT WriterProperties { pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, size_statistics_level_, std::move(file_encryption_properties_), default_column_properties_, column_properties, data_page_version_, - store_decimal_as_integer_, std::move(sorting_columns_), + min_space_savings_, store_decimal_as_integer_, std::move(sorting_columns_), content_defined_chunking_enabled_, content_defined_chunking_options_)); } @@ -933,6 +946,7 @@ class PARQUET_EXPORT WriterProperties { int64_t max_rows_per_page_; ParquetVersion::type version_; ParquetDataPageVersion data_page_version_; + std::optional min_space_savings_ = {}; std::string created_by_; bool store_decimal_as_integer_; bool page_checksum_enabled_; @@ -973,6 +987,9 @@ class PARQUET_EXPORT WriterProperties { return parquet_data_page_version_; } + + std::optional min_space_savings() const { return min_space_savings_; } + inline ParquetVersion::type version() const { return parquet_version_; } inline std::string created_by() const { return parquet_created_by_; } @@ -1100,7 +1117,8 @@ class PARQUET_EXPORT WriterProperties { std::shared_ptr file_encryption_properties, const ColumnProperties& default_column_properties, const std::unordered_map& column_properties, - ParquetDataPageVersion data_page_version, bool store_short_decimal_as_integer, + ParquetDataPageVersion data_page_version, std::optional min_space_savings, + bool store_short_decimal_as_integer, std::vector sorting_columns, bool content_defined_chunking_enabled, CdcOptions content_defined_chunking_options) : pool_(pool), @@ -1110,6 +1128,7 @@ class PARQUET_EXPORT WriterProperties { pagesize_(pagesize), max_rows_per_page_(max_rows_per_page), parquet_data_page_version_(data_page_version), + min_space_savings_(min_space_savings), parquet_version_(version), parquet_created_by_(created_by), store_decimal_as_integer_(store_short_decimal_as_integer), @@ -1129,6 +1148,7 @@ class PARQUET_EXPORT WriterProperties { int64_t pagesize_; int64_t max_rows_per_page_; ParquetDataPageVersion parquet_data_page_version_; + std::optional min_space_savings_; ParquetVersion::type parquet_version_; std::string parquet_created_by_; bool store_decimal_as_integer_; diff --git a/cpp/src/parquet/properties_test.cc b/cpp/src/parquet/properties_test.cc index 2f22844f8bbf..f56d38eba619 100644 --- a/cpp/src/parquet/properties_test.cc +++ b/cpp/src/parquet/properties_test.cc @@ -80,6 +80,33 @@ TEST(TestWriterProperties, DefaultCompression) { ::arrow::util::kUseDefaultCompressionLevel); } +// checks if WriterProperties correctly store, copy, and clear min_space_savings +TEST(TestWriterProperties, MinSpaceSavings) { + WriterProperties::Builder builder; + ASSERT_FALSE(builder.build()->min_space_savings().has_value()); + + for (double savings : {0.0, 0.1, 1.0}) { + auto properties = builder.min_space_savings(savings)->build(); + ASSERT_EQ(std::optional(savings), properties->min_space_savings()); + + auto copied = WriterProperties::Builder(*properties).build(); + ASSERT_EQ(properties->min_space_savings(), copied->min_space_savings()); + } + + auto properties = builder.min_space_savings(std::nullopt)->build(); + ASSERT_FALSE(properties->min_space_savings().has_value()); +} + +// Tests for invalid settings +TEST(TestWriterProperties, InvalidMinSpaceSavings) { + for (double savings : {-0.1, 1.1, std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN()}) { + EXPECT_THROW(WriterProperties::Builder().min_space_savings(savings)->build(), + ParquetException); + } +} + TEST(TestWriterProperties, AdvancedHandling) { WriterProperties::Builder builder; builder.compression("gzip", Compression::GZIP); From 01512cc6cec1e2bc1db21bd7eb40b1f201d72af8 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Fri, 9 Oct 2026 11:37:50 -0500 Subject: [PATCH 02/15] Let Python users set minimum compression savings Add an option to choose how much space compression must save when writing Parquet files from Python. Add tests as well. --- python/pyarrow/_dataset_parquet.pyx | 2 + python/pyarrow/_parquet.pxd | 3 +- python/pyarrow/_parquet.pyx | 16 +++++-- python/pyarrow/includes/libparquet.pxd | 1 + python/pyarrow/parquet/core.py | 4 ++ .../tests/parquet/test_parquet_writer.py | 44 +++++++++++++++++++ python/pyarrow/tests/test_dataset.py | 21 +++++++++ 7 files changed, 87 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index 43eca8957be8..ffff69f4bcfe 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -655,6 +655,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): ), column_encoding=self._properties["column_encoding"], data_page_version=self._properties["data_page_version"], + min_space_savings=self._properties["min_space_savings"], encryption_properties=self._properties["encryption_properties"], write_batch_size=self._properties["write_batch_size"], dictionary_pagesize_limit=self._properties["dictionary_pagesize_limit"], @@ -707,6 +708,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): use_byte_stream_split=False, column_encoding=None, data_page_version="1.0", + min_space_savings=None, use_deprecated_int96_timestamps=False, coerce_timestamps=None, allow_truncated_timestamps=False, diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 36fc2ccf2f33..28a5c676ec98 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -57,7 +57,8 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( sorting_columns=*, store_decimal_as_integer=*, use_content_defined_chunking=*, - bloom_filter_options=* + bloom_filter_options=*, + min_space_savings=* ) except * diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 2621c3bd6d61..640a69a2453d 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -2076,13 +2076,21 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( sorting_columns=None, store_decimal_as_integer=False, use_content_defined_chunking=False, - bloom_filter_options=None) except *: + bloom_filter_options=None, + min_space_savings=None) except *: """General writer properties""" cdef: shared_ptr[WriterProperties] properties WriterProperties.Builder props CdcOptions cdc_options + double savings + + if min_space_savings is not None: + savings = min_space_savings + if not (0.0 <= savings <= 1.0): + raise ValueError("min_space_savings must be in range [0, 1]") + props.min_space_savings(optional[double](savings)) # data_page_version @@ -2405,7 +2413,8 @@ cdef class ParquetWriter(_Weakrefable): store_decimal_as_integer=False, use_content_defined_chunking=False, write_time_adjusted_to_utc=False, - bloom_filter_options=None): + bloom_filter_options=None, + min_space_savings=None): cdef: shared_ptr[WriterProperties] properties shared_ptr[ArrowWriterProperties] arrow_properties @@ -2442,7 +2451,8 @@ cdef class ParquetWriter(_Weakrefable): sorting_columns=sorting_columns, store_decimal_as_integer=store_decimal_as_integer, use_content_defined_chunking=use_content_defined_chunking, - bloom_filter_options=bloom_filter_options + bloom_filter_options=bloom_filter_options, + min_space_savings=min_space_savings, ) arrow_properties = _create_arrow_writer_properties( use_deprecated_int96_timestamps=use_deprecated_int96_timestamps, diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index 3a5f91716c17..c95d81a26761 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -485,6 +485,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil: cdef cppclass WriterProperties: cppclass Builder: Builder* data_page_version(ParquetDataPageVersion version) + Builder* min_space_savings(optional[double] min_space_savings) Builder* version(ParquetVersion version) Builder* compression(ParquetCompression codec) Builder* compression(const c_string& path, diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 4de88fe22997..bccf47b917ed 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -1090,6 +1090,7 @@ def __init__(self, where, schema, filesystem=None, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking=False, + min_space_savings=None, **options): if use_deprecated_int96_timestamps is None: # Use int96 timestamps for Spark @@ -1147,6 +1148,7 @@ def __init__(self, where, schema, filesystem=None, max_rows_per_page=max_rows_per_page, bloom_filter_options=bloom_filter_options, use_content_defined_chunking=use_content_defined_chunking, + min_space_savings=min_space_savings, **options) self.is_open = True @@ -2039,6 +2041,7 @@ def write_table(table, where, row_group_size=None, version='2.6', max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking=False, + min_space_savings=None, **kwargs): # Implementor's note: when adding keywords here / updating defaults, also # update it in write_to_dataset and _dataset_parquet.pyx ParquetFileWriteOptions @@ -2074,6 +2077,7 @@ def write_table(table, where, row_group_size=None, version='2.6', max_rows_per_page=max_rows_per_page, bloom_filter_options=bloom_filter_options, use_content_defined_chunking=use_content_defined_chunking, + min_space_savings=min_space_savings, **kwargs) as writer: writer.write_table(table, row_group_size=row_group_size) except Exception: diff --git a/python/pyarrow/tests/parquet/test_parquet_writer.py b/python/pyarrow/tests/parquet/test_parquet_writer.py index a49441f09f45..89b8eca78f6d 100644 --- a/python/pyarrow/tests/parquet/test_parquet_writer.py +++ b/python/pyarrow/tests/parquet/test_parquet_writer.py @@ -41,6 +41,50 @@ pytestmark = pytest.mark.parquet +@pytest.mark.parametrize("minimum", [None, 0.0, 0.1, 1.0]) +@pytest.mark.parametrize("page_version", ["1.0", "2.0"]) +@pytest.mark.parametrize("compression", ["none", "snappy"]) +@pytest.mark.parametrize("use_writer", [False, True]) +def test_min_space_savings( + minimum, page_version, compression, use_writer): + table = pa.table({ + "value": pa.array([0.0] * 1000, type=pa.float32()) + }) + sink = pa.BufferOutputStream() + options = dict( + compression=compression, + use_dictionary=False, + data_page_version=page_version, + min_space_savings=minimum, + ) + + if use_writer: + with pq.ParquetWriter(sink, table.schema, **options) as writer: + writer.write_table(table) + else: + pq.write_table(table, sink, **options) + + assert pq.read_table(sink.getvalue()).equals(table) + +# Tests invalid settings +@pytest.mark.parametrize( + "minimum", + [-0.1, 1.1, float("inf"), -float("inf"), float("nan")] +) +@pytest.mark.parametrize("use_writer", [False, True]) +def test_invalid_min_space_savings(minimum, use_writer): + table = pa.table({"value": [1.0]}) + sink = pa.BufferOutputStream() + + with pytest.raises(ValueError, match="min_space_savings"): + if use_writer: + pq.ParquetWriter( + sink, table.schema, min_space_savings=minimum) + else: + pq.write_table( + table, sink, min_space_savings=minimum) + + @pytest.mark.pandas def test_parquet_incremental_file_build(tempdir): df = _test_dataframe(100) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 0a94c0bd9875..7daf2dfb2558 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -5949,6 +5949,27 @@ def test_write_dataset_content_defined_chunking(tempdir, cdc): assert ds.dataset(tempdir, format=fmt).to_table().equals(expected_table) +@pytest.mark.parquet +@pytest.mark.parametrize("minimum", [None, 0.0, 0.1, 1.0]) +def test_parquet_min_space_savings(tempdir, minimum): + import pyarrow.parquet as pq + + table = pa.table({ + "value": pa.array([0.0] * 1000, type=pa.float32()) + }) + pq.write_to_dataset( + table, + tempdir, + compression="snappy", + use_dictionary=False, + data_page_version="2.0", + min_space_savings=minimum, + ) + + result = ds.dataset(tempdir, format="parquet").to_table() + assert result.equals(table) + + def test_make_write_options_error(): # GH-39440: calling make_write_options as a static class method msg_1 = ("make_write_options() should be called on an " From 89bf1b25ce6d4c6f667a86de309a78625dda1e35 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sat, 10 Oct 2026 04:54:01 -0500 Subject: [PATCH 03/15] Fix ignored compression savings in dataset writing Dataset writing ignored min_space_savings. Move the argument to the end of the call so it reaches the writer. Add a test to check that the setting changes the file size. --- python/pyarrow/_dataset_parquet.pyx | 2 +- python/pyarrow/tests/test_dataset.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index ffff69f4bcfe..4b9d2e2cc8ee 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -655,7 +655,6 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): ), column_encoding=self._properties["column_encoding"], data_page_version=self._properties["data_page_version"], - min_space_savings=self._properties["min_space_savings"], encryption_properties=self._properties["encryption_properties"], write_batch_size=self._properties["write_batch_size"], dictionary_pagesize_limit=self._properties["dictionary_pagesize_limit"], @@ -665,6 +664,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): store_decimal_as_integer=self._properties["store_decimal_as_integer"], use_content_defined_chunking=self._properties["use_content_defined_chunking"], bloom_filter_options=self._properties["bloom_filter_options"], + min_space_savings=self._properties["min_space_savings"], ) def _set_arrow_properties(self): diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 7daf2dfb2558..589bc08f037c 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -5970,6 +5970,26 @@ def test_parquet_min_space_savings(tempdir, minimum): assert result.equals(table) +@pytest.mark.parquet +def test_dataset_min_space_savings_changes_size(tempdir): + table = pa.table({"value": [0.0] * 1000}) + sizes = [] + + for minimum in [None, 1.0]: + directory = tempdir / str(minimum) + pq.write_to_dataset( + table, + directory, + compression="snappy", + use_dictionary=False, + data_page_version="2.0", + min_space_savings=minimum, + ) + sizes.append(next(directory.iterdir()).stat().st_size) + + assert sizes[1] > sizes[0] + + def test_make_write_options_error(): # GH-39440: calling make_write_options as a static class method msg_1 = ("make_write_options() should be called on an " From 7d705a931c8c109f8cab6abe2818e8c1aa34e54b Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 18 Oct 2026 09:12:41 -0500 Subject: [PATCH 04/15] Reject min_space_savings for V1 pages Throw an error when users set min_space_savings for V1 pages. Update C++ and Python tests. --- cpp/src/parquet/properties.h | 3 ++ cpp/src/parquet/properties_test.cc | 23 ++++++++++++++- python/pyarrow/_parquet.pyx | 2 ++ .../tests/parquet/test_parquet_writer.py | 28 ++++++++++++++++--- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index 3bb721c7b560..4771b1e3b3a7 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -896,6 +896,9 @@ class PARQUET_EXPORT WriterProperties { !(*min_space_savings_ >= 0.0 && *min_space_savings_ <= 1.0)) { throw ParquetException("min_space_savings must be in range [0, 1]"); } + if (min_space_savings_ && data_page_version_ == ParquetDataPageVersion::V1) { + throw ParquetException("min_space_savings requires V2 data pages"); + } std::unordered_map column_properties; auto get = [&](const std::string& key) -> ColumnProperties& { auto it = column_properties.find(key); diff --git a/cpp/src/parquet/properties_test.cc b/cpp/src/parquet/properties_test.cc index f56d38eba619..19158505e97c 100644 --- a/cpp/src/parquet/properties_test.cc +++ b/cpp/src/parquet/properties_test.cc @@ -85,6 +85,7 @@ TEST(TestWriterProperties, MinSpaceSavings) { WriterProperties::Builder builder; ASSERT_FALSE(builder.build()->min_space_savings().has_value()); + builder.data_page_version(ParquetDataPageVersion::V2); for (double savings : {0.0, 0.1, 1.0}) { auto properties = builder.min_space_savings(savings)->build(); ASSERT_EQ(std::optional(savings), properties->min_space_savings()); @@ -102,11 +103,31 @@ TEST(TestWriterProperties, InvalidMinSpaceSavings) { for (double savings : {-0.1, 1.1, std::numeric_limits::infinity(), -std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { - EXPECT_THROW(WriterProperties::Builder().min_space_savings(savings)->build(), + EXPECT_THROW(WriterProperties::Builder() + .data_page_version(ParquetDataPageVersion::V2) + ->min_space_savings(savings) + ->build(), ParquetException); } } +TEST(TestWriterProperties, MinSpaceSavingsRequiresDataPageV2) { + for (double savings : {0.0, 0.1, 1.0}) { + WriterProperties::Builder builder; + builder.min_space_savings(savings); + EXPECT_THROW(builder.build(), ParquetException); + + builder.data_page_version(ParquetDataPageVersion::V2); + EXPECT_NO_THROW(builder.build()); + + builder.data_page_version(ParquetDataPageVersion::V1); + EXPECT_THROW(builder.build(), ParquetException); + + builder.min_space_savings(std::nullopt); + EXPECT_NO_THROW(builder.build()); + } +} + TEST(TestWriterProperties, AdvancedHandling) { WriterProperties::Builder builder; builder.compression("gzip", Compression::GZIP); diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 640a69a2453d..9574b9c06b79 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -2090,6 +2090,8 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( savings = min_space_savings if not (0.0 <= savings <= 1.0): raise ValueError("min_space_savings must be in range [0, 1]") + if data_page_version in (None, "1.0"): + raise ValueError("min_space_savings requires V2 data pages") props.min_space_savings(optional[double](savings)) # data_page_version diff --git a/python/pyarrow/tests/parquet/test_parquet_writer.py b/python/pyarrow/tests/parquet/test_parquet_writer.py index 89b8eca78f6d..2ae186811dfd 100644 --- a/python/pyarrow/tests/parquet/test_parquet_writer.py +++ b/python/pyarrow/tests/parquet/test_parquet_writer.py @@ -41,8 +41,13 @@ pytestmark = pytest.mark.parquet -@pytest.mark.parametrize("minimum", [None, 0.0, 0.1, 1.0]) -@pytest.mark.parametrize("page_version", ["1.0", "2.0"]) +@pytest.mark.parametrize("page_version, minimum", [ + ("1.0", None), + ("2.0", None), + ("2.0", 0.0), + ("2.0", 0.1), + ("2.0", 1.0), +]) @pytest.mark.parametrize("compression", ["none", "snappy"]) @pytest.mark.parametrize("use_writer", [False, True]) def test_min_space_savings( @@ -79,10 +84,25 @@ def test_invalid_min_space_savings(minimum, use_writer): with pytest.raises(ValueError, match="min_space_savings"): if use_writer: pq.ParquetWriter( - sink, table.schema, min_space_savings=minimum) + sink, table.schema, data_page_version="2.0", + min_space_savings=minimum) else: pq.write_table( - table, sink, min_space_savings=minimum) + table, sink, data_page_version="2.0", + min_space_savings=minimum) + + +@pytest.mark.parametrize("minimum", [0.0, 0.1, 1.0]) +@pytest.mark.parametrize("use_writer", [False, True]) +def test_min_space_savings_rejects_v1(minimum, use_writer): + table = pa.table({"value": [1.0]}) + sink = pa.BufferOutputStream() + + with pytest.raises(ValueError, match="min_space_savings requires V2"): + if use_writer: + pq.ParquetWriter(sink, table.schema, min_space_savings=minimum) + else: + pq.write_table(table, sink, min_space_savings=minimum) @pytest.mark.pandas From d8d4642330d9e3edf513dce7ca8538ca88ea159b Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 18 Oct 2026 09:21:53 -0500 Subject: [PATCH 05/15] Skip compression when it saves no space Require compressed data to be smaller, even when min_space_savings is 0.0. Simplify the compression check and update the test. --- cpp/src/parquet/column_writer.cc | 16 +++++----------- cpp/src/parquet/column_writer_test.cc | 1 + 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 9d8026c3cc08..251ea8d3b3d0 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -1058,17 +1058,11 @@ void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size, if (pager_->has_compressor() && values->size() > 0) { pager_->Compress(*values, compressor_temp_buffer_.get()); - if (const auto minimum = properties_->min_space_savings()) { - - //Checks if compressed size meets the minimum threshold - const double savings = - 1.0 - static_cast(compressor_temp_buffer_->size()) / values->size(); - page_is_compressed = savings >= *minimum; - } else { - // Keeps original behavior - // Allows users keep min_space_savings unset - page_is_compressed = compressor_temp_buffer_->size() < values->size(); - } + const int64_t compressed = compressor_temp_buffer_->size(); + const auto minimum = properties_->min_space_savings(); + page_is_compressed = + compressed < values->size() && + (!minimum || 1.0 - static_cast(compressed) / values->size() >= *minimum); } std::shared_ptr compressed_values = (page_is_compressed ? compressor_temp_buffer_ : values); diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index a65facf4e8a4..4ceb5dc40890 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -2106,6 +2106,7 @@ TEST_F(TestValuesWriterInt32Type, MinSpaceSavingsDataPageV2) { const std::vector, bool>> cases = { {std::nullopt, true}, + {0.0, true}, {0.1, true}, {1.0, false}, }; From f4cc42733425f366c44398f2c511cece0689f993 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 18 Oct 2026 09:38:11 -0500 Subject: [PATCH 06/15] Reject bad savings values right away Check min_space_savings when users set it. Update the test and Python error handling. --- cpp/src/parquet/properties.h | 9 ++++----- cpp/src/parquet/properties_test.cc | 5 +---- python/pyarrow/includes/libparquet.pxd | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index 4771b1e3b3a7..1ca670bfad5d 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -533,6 +533,10 @@ class PARQUET_EXPORT WriterProperties { /// Used to set minimum threshold for V2 page data values compression /// If min_space_savings unset, keep compression if compressed value is smaller. Builder* min_space_savings(std::optional min_space_savings) { + if (min_space_savings && + !(*min_space_savings >= 0.0 && *min_space_savings <= 1.0)) { + throw ParquetException("min_space_savings must be in range [0, 1]"); + } min_space_savings_ = min_space_savings; return this; } @@ -891,11 +895,6 @@ class PARQUET_EXPORT WriterProperties { /// \brief Build the WriterProperties with the builder parameters. /// \return The WriterProperties defined by the builder. std::shared_ptr build() { - // checks for invalid input of min_space_savings - if (min_space_savings_ && - !(*min_space_savings_ >= 0.0 && *min_space_savings_ <= 1.0)) { - throw ParquetException("min_space_savings must be in range [0, 1]"); - } if (min_space_savings_ && data_page_version_ == ParquetDataPageVersion::V1) { throw ParquetException("min_space_savings requires V2 data pages"); } diff --git a/cpp/src/parquet/properties_test.cc b/cpp/src/parquet/properties_test.cc index 19158505e97c..5758bd3e3b13 100644 --- a/cpp/src/parquet/properties_test.cc +++ b/cpp/src/parquet/properties_test.cc @@ -103,10 +103,7 @@ TEST(TestWriterProperties, InvalidMinSpaceSavings) { for (double savings : {-0.1, 1.1, std::numeric_limits::infinity(), -std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { - EXPECT_THROW(WriterProperties::Builder() - .data_page_version(ParquetDataPageVersion::V2) - ->min_space_savings(savings) - ->build(), + EXPECT_THROW(WriterProperties::Builder().min_space_savings(savings), ParquetException); } } diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index c95d81a26761..ce92aa8c7038 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -485,7 +485,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil: cdef cppclass WriterProperties: cppclass Builder: Builder* data_page_version(ParquetDataPageVersion version) - Builder* min_space_savings(optional[double] min_space_savings) + Builder* min_space_savings(optional[double] min_space_savings) except + Builder* version(ParquetVersion version) Builder* compression(ParquetCompression codec) Builder* compression(const c_string& path, From f20f66c7f50062b46c3d33b7565a24de42fba37e Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 18 Oct 2026 09:43:05 -0500 Subject: [PATCH 07/15] Check that Python uses compression savings Check compressed and original sizes in the existing test. Keep checking that the data reads back correctly. --- .../pyarrow/tests/parquet/test_parquet_writer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/parquet/test_parquet_writer.py b/python/pyarrow/tests/parquet/test_parquet_writer.py index 2ae186811dfd..1bf7ad74f0bd 100644 --- a/python/pyarrow/tests/parquet/test_parquet_writer.py +++ b/python/pyarrow/tests/parquet/test_parquet_writer.py @@ -69,7 +69,19 @@ def test_min_space_savings( else: pq.write_table(table, sink, **options) - assert pq.read_table(sink.getvalue()).equals(table) + buf = sink.getvalue() + assert pq.read_table(buf).equals(table) + + column = pq.ParquetFile(buf).metadata.row_group(0).column(0) + expect_uncompressed = ( + compression == "none" + or (page_version == "2.0" and minimum == 1.0) + ) + if expect_uncompressed: + assert column.total_compressed_size == column.total_uncompressed_size + else: + assert column.total_compressed_size < column.total_uncompressed_size + # Tests invalid settings @pytest.mark.parametrize( From adef6d4fed2b51ebe13a9ed24bbecdde86d78db5 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sun, 18 Oct 2026 12:44:14 -0500 Subject: [PATCH 08/15] Explain minimum compression savings Explain min_space_savings in the Python and C++ docs. --- docs/source/cpp/parquet.rst | 7 +++++++ python/pyarrow/parquet/core.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/source/cpp/parquet.rst b/docs/source/cpp/parquet.rst index ac385668663d..58ef0036d889 100644 --- a/docs/source/cpp/parquet.rst +++ b/docs/source/cpp/parquet.rst @@ -271,6 +271,13 @@ with ``data_page_version``. It's V1 by default; setting to V2 will allow more optimal compression (skipping compressing pages where there isn't a space benefit), but not all readers support this data page version. +For V2 pages, use ``min_space_savings`` to set how much space compression must +save. Use a value from 0.0 to 1.0. For example, ``min_space_savings(0.1)`` requires +at least 10% savings. If the space savings do not meet the threshold, it is not +compressed. If the ``min_space_savings`` option is unset, compression is kept as +long as compressed values are smaller than original values. Setting a number +with V1 or using a number outside the allowed range throws ``ParquetException``. + Compression is off by default, but to get the most out of Parquet, you should also choose a compression codec. You can choose one for the whole file or choose one for individual columns. If you choose a mix, the file-level option diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index bccf47b917ed..5f8d79d87335 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -840,6 +840,14 @@ def _sanitize_table(table, new_schema, flavor): The serialized Parquet data page format version to write, defaults to 1.0. This does not impact the file schema logical types and Arrow to Parquet type casting behavior; for that use the "version" option. +min_space_savings : float or None, default None + Minimum space savings required to keep compression for each V2 data page. + Use a value from 0.0 to 1.0. For example, 0.1 requires at least 10% savings. + Savings are measured on encoded values, excluding repetition and definition + levels. Compression is kept only if it makes the values smaller. + If None, no extra minimum savings is required. + Setting a number requires ``data_page_version="2.0"``. Using V1 or a number + outside the allowed range raises ``ValueError``. use_compliant_nested_type : bool, default True Whether to write compliant Parquet nested type (lists) as defined `here Date: Sun, 18 Oct 2026 13:27:05 -0500 Subject: [PATCH 09/15] Fix C++ formatting Fix spacing and line wrapping in the C++ changes. --- cpp/src/parquet/column_writer_test.cc | 18 ++++++++---------- cpp/src/parquet/properties.h | 6 ++---- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index 4ceb5dc40890..cf5312dce257 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -2094,7 +2094,6 @@ TEST_F(TestValuesWriterInt32Type, AvoidCompressedInDataPageV2) { // - min_space_savings = 0.1 // - min_space_savings = 1 TEST_F(TestValuesWriterInt32Type, MinSpaceSavingsDataPageV2) { - // Used zeros because it compresses well this->SetUpSchema(Repetition::OPTIONAL); this->descr_ = this->schema_.Column(0); @@ -2112,20 +2111,19 @@ TEST_F(TestValuesWriterInt32Type, MinSpaceSavingsDataPageV2) { }; for (const auto& [minimum, expected_compressed] : cases) { - auto writer = this->BuildWriter( - SMALL_SIZE, column_properties, ParquetVersion::PARQUET_2_LATEST, - ParquetDataPageVersion::V2, false, kDefaultDataPageSize, - kDefaultMaxRowsPerPage, minimum); + auto writer = + this->BuildWriter(SMALL_SIZE, column_properties, ParquetVersion::PARQUET_2_LATEST, + ParquetDataPageVersion::V2, false, kDefaultDataPageSize, + kDefaultMaxRowsPerPage, minimum); - writer->WriteBatch(SMALL_SIZE, this->def_levels_.data(), nullptr, - this->values_ptr_); + writer->WriteBatch(SMALL_SIZE, this->def_levels_.data(), nullptr, this->values_ptr_); writer->Close(); ASSERT_OK_AND_ASSIGN(auto buffer, this->sink_->Finish()); - auto page_reader = PageReader::Open( - std::make_shared<::arrow::io::BufferReader>(buffer), SMALL_SIZE, - Compression::ZSTD, default_reader_properties(), *this->descr_); + auto page_reader = + PageReader::Open(std::make_shared<::arrow::io::BufferReader>(buffer), SMALL_SIZE, + Compression::ZSTD, default_reader_properties(), *this->descr_); auto page = page_reader->NextPage(); diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index 1ca670bfad5d..aa6119549c08 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -989,7 +989,6 @@ class PARQUET_EXPORT WriterProperties { return parquet_data_page_version_; } - std::optional min_space_savings() const { return min_space_savings_; } inline ParquetVersion::type version() const { return parquet_version_; } @@ -1120,9 +1119,8 @@ class PARQUET_EXPORT WriterProperties { const ColumnProperties& default_column_properties, const std::unordered_map& column_properties, ParquetDataPageVersion data_page_version, std::optional min_space_savings, - bool store_short_decimal_as_integer, - std::vector sorting_columns, bool content_defined_chunking_enabled, - CdcOptions content_defined_chunking_options) + bool store_short_decimal_as_integer, std::vector sorting_columns, + bool content_defined_chunking_enabled, CdcOptions content_defined_chunking_options) : pool_(pool), dictionary_pagesize_limit_(dictionary_pagesize_limit), write_batch_size_(write_batch_size), From 98a8f31115831b6f51039adb3303c23ea4678a8c Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Sat, 24 Oct 2026 12:12:50 -0500 Subject: [PATCH 10/15] Fix documentation for rok's doc comment --- cpp/src/parquet/properties.h | 2 +- python/pyarrow/parquet/core.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index aa6119549c08..df22381ab209 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -530,7 +530,7 @@ class PARQUET_EXPORT WriterProperties { return this; } - /// Used to set minimum threshold for V2 page data values compression + /// Used to set minimum compression ratio threshold for V2 page data values compression /// If min_space_savings unset, keep compression if compressed value is smaller. Builder* min_space_savings(std::optional min_space_savings) { if (min_space_savings && diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 5f8d79d87335..6b1502ef33b0 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -840,14 +840,6 @@ def _sanitize_table(table, new_schema, flavor): The serialized Parquet data page format version to write, defaults to 1.0. This does not impact the file schema logical types and Arrow to Parquet type casting behavior; for that use the "version" option. -min_space_savings : float or None, default None - Minimum space savings required to keep compression for each V2 data page. - Use a value from 0.0 to 1.0. For example, 0.1 requires at least 10% savings. - Savings are measured on encoded values, excluding repetition and definition - levels. Compression is kept only if it makes the values smaller. - If None, no extra minimum savings is required. - Setting a number requires ``data_page_version="2.0"``. Using V1 or a number - outside the allowed range raises ``ValueError``. use_compliant_nested_type : bool, default True Whether to write compliant Parquet nested type (lists) as defined `here Date: Sat, 24 Oct 2026 20:05:13 -0500 Subject: [PATCH 11/15] Add compression savings threshold test --- cpp/src/parquet/properties.h | 5 ++-- .../tests/parquet/test_parquet_writer.py | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index df22381ab209..3700a84f0ff7 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -530,8 +530,9 @@ class PARQUET_EXPORT WriterProperties { return this; } - /// Used to set minimum compression ratio threshold for V2 page data values compression - /// If min_space_savings unset, keep compression if compressed value is smaller. + /// Used to set minimum compression ratio threshold for V2 page data values + /// compression. If min_space_savings unset, keep compression if compressed value is + /// smaller. Builder* min_space_savings(std::optional min_space_savings) { if (min_space_savings && !(*min_space_savings >= 0.0 && *min_space_savings <= 1.0)) { diff --git a/python/pyarrow/tests/parquet/test_parquet_writer.py b/python/pyarrow/tests/parquet/test_parquet_writer.py index 1bf7ad74f0bd..0281cd362f2c 100644 --- a/python/pyarrow/tests/parquet/test_parquet_writer.py +++ b/python/pyarrow/tests/parquet/test_parquet_writer.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import random + import pytest import pyarrow as pa @@ -83,6 +85,30 @@ def test_min_space_savings( assert column.total_compressed_size < column.total_uncompressed_size +def _v2_column_chunk(table, **options): + sink = pa.BufferOutputStream() + pq.write_table(table, sink, compression="snappy", use_dictionary=False, + data_page_version="2.0", **options) + return pq.ParquetFile(sink.getvalue()).metadata.row_group(0).column(0) + + +def test_min_space_savings_threshold(): + rng = random.Random(42) + values = [rng.randrange(16) for _ in range(1000)] + table = pa.table({"v": pa.array(values, pa.int32())}) + + base = _v2_column_chunk(table) + savings = 1 - base.total_compressed_size / base.total_uncompressed_size + assert 0.1 < savings < 0.9 + + kept = _v2_column_chunk(table, min_space_savings=savings - 0.05) + assert kept.total_compressed_size == base.total_compressed_size + + dropped = _v2_column_chunk(table, min_space_savings=savings + 0.05) + assert dropped.total_compressed_size == dropped.total_uncompressed_size + assert dropped.compression == "SNAPPY" + + # Tests invalid settings @pytest.mark.parametrize( "minimum", From 6ea5a992bcb815dcad5bff0f8bb76d5548df4660 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Mon, 26 Oct 2026 14:02:20 -0500 Subject: [PATCH 12/15] Test both sides of the compression savings threshold --- cpp/src/parquet/column_writer_test.cc | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index cf5312dce257..1af99b7cc8d3 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -29,6 +29,7 @@ #include "arrow/testing/gtest_util.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_builders.h" +#include "arrow/util/compression.h" #include "arrow/util/config.h" #include "arrow/util/key_value_metadata.h" @@ -2089,25 +2090,37 @@ TEST_F(TestValuesWriterInt32Type, AvoidCompressedInDataPageV2) { } } -// Tests if compression works when: -// - min_space_savings is unset -// - min_space_savings = 0.1 -// - min_space_savings = 1 +// Tests the default, fixed thresholds, and both sides of the actual savings. TEST_F(TestValuesWriterInt32Type, MinSpaceSavingsDataPageV2) { // Used zeros because it compresses well this->SetUpSchema(Repetition::OPTIONAL); this->descr_ = this->schema_.Column(0); this->GenerateData(SMALL_SIZE); std::fill(this->values_.begin(), this->values_.end(), 0); + std::fill(this->def_levels_.begin(), this->def_levels_.end(), 1); ColumnProperties column_properties; column_properties.set_compression(Compression::ZSTD); + // PLAIN-encoded int32 zeros have the same bytes as the input values. + auto codec = GetCodec(Compression::ZSTD); + const int64_t values_size = SMALL_SIZE * sizeof(int32_t); + const auto* values_bytes = reinterpret_cast(this->values_ptr_); + std::vector compressed(codec->MaxCompressedLen(values_size, values_bytes)); + ASSERT_OK_AND_ASSIGN( + auto compressed_size, + codec->Compress(values_size, values_bytes, compressed.size(), compressed.data())); + const double savings = 1.0 - static_cast(compressed_size) / values_size; + ASSERT_GT(savings, 0.01); + ASSERT_LT(savings, 0.99); + const std::vector, bool>> cases = { {std::nullopt, true}, {0.0, true}, {0.1, true}, {1.0, false}, + {savings - 0.01, true}, + {savings + 0.01, false}, }; for (const auto& [minimum, expected_compressed] : cases) { From 68ead0d915a0d241f47de53e4a021da451409fd8 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Mon, 26 Oct 2026 14:53:05 -0500 Subject: [PATCH 13/15] Rename compression savings and use zero as the default Allow zero with V1 and reject nonzero values. Combine dataset tests and keep threshold checks in C++. Update docs to match. --- cpp/src/parquet/column_writer.cc | 4 +- cpp/src/parquet/column_writer_test.cc | 9 ++-- cpp/src/parquet/properties.h | 34 ++++++------- cpp/src/parquet/properties_test.cc | 28 +++++------ docs/source/cpp/parquet.rst | 10 ++-- python/pyarrow/_dataset_parquet.pyx | 4 +- python/pyarrow/_parquet.pxd | 2 +- python/pyarrow/_parquet.pyx | 19 ++++--- python/pyarrow/includes/libparquet.pxd | 2 +- python/pyarrow/parquet/core.py | 16 +++--- .../tests/parquet/test_parquet_writer.py | 50 +++++-------------- python/pyarrow/tests/test_dataset.py | 27 +++------- 12 files changed, 82 insertions(+), 123 deletions(-) diff --git a/cpp/src/parquet/column_writer.cc b/cpp/src/parquet/column_writer.cc index 251ea8d3b3d0..feab397bd306 100644 --- a/cpp/src/parquet/column_writer.cc +++ b/cpp/src/parquet/column_writer.cc @@ -1059,10 +1059,10 @@ void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size, pager_->Compress(*values, compressor_temp_buffer_.get()); const int64_t compressed = compressor_temp_buffer_->size(); - const auto minimum = properties_->min_space_savings(); + const auto minimum = properties_->min_compression_savings(); page_is_compressed = compressed < values->size() && - (!minimum || 1.0 - static_cast(compressed) / values->size() >= *minimum); + 1.0 - static_cast(compressed) / values->size() >= minimum; } std::shared_ptr compressed_values = (page_is_compressed ? compressor_temp_buffer_ : values); diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index 1af99b7cc8d3..aa2e65bf6628 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -116,7 +116,7 @@ class TestPrimitiveWriter : public PrimitiveTypedTest { const ParquetDataPageVersion data_page_version = ParquetDataPageVersion::V1, bool enable_checksum = false, int64_t page_size = kDefaultDataPageSize, int64_t max_rows_per_page = kDefaultMaxRowsPerPage, - std::optional min_space_savings = {}) { + double min_compression_savings = 0.0) { sink_ = CreateOutputStream(); WriterProperties::Builder wp_builder; wp_builder.version(version)->data_page_version(data_page_version); @@ -134,7 +134,7 @@ class TestPrimitiveWriter : public PrimitiveTypedTest { wp_builder.max_statistics_size(column_properties.max_statistics_size()); wp_builder.data_pagesize(page_size); wp_builder.max_rows_per_page(max_rows_per_page); - wp_builder.min_space_savings(min_space_savings); + wp_builder.min_compression_savings(min_compression_savings); writer_properties_ = wp_builder.build(); metadata_ = ColumnChunkMetaDataBuilder::Make(writer_properties_, this->descr_); @@ -2091,7 +2091,7 @@ TEST_F(TestValuesWriterInt32Type, AvoidCompressedInDataPageV2) { } // Tests the default, fixed thresholds, and both sides of the actual savings. -TEST_F(TestValuesWriterInt32Type, MinSpaceSavingsDataPageV2) { +TEST_F(TestValuesWriterInt32Type, MinCompressionSavingsDataPageV2) { // Used zeros because it compresses well this->SetUpSchema(Repetition::OPTIONAL); this->descr_ = this->schema_.Column(0); @@ -2114,8 +2114,7 @@ TEST_F(TestValuesWriterInt32Type, MinSpaceSavingsDataPageV2) { ASSERT_GT(savings, 0.01); ASSERT_LT(savings, 0.99); - const std::vector, bool>> cases = { - {std::nullopt, true}, + const std::vector> cases = { {0.0, true}, {0.1, true}, {1.0, false}, diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h index 3700a84f0ff7..3a1e8bd19e14 100644 --- a/cpp/src/parquet/properties.h +++ b/cpp/src/parquet/properties.h @@ -403,7 +403,7 @@ class PARQUET_EXPORT WriterProperties { max_rows_per_page_(properties.max_rows_per_page()), version_(properties.version()), data_page_version_(properties.data_page_version()), - min_space_savings_(properties.min_space_savings()), + min_compression_savings_(properties.min_compression_savings()), created_by_(properties.created_by()), store_decimal_as_integer_(properties.store_decimal_as_integer()), page_checksum_enabled_(properties.page_checksum_enabled()), @@ -531,14 +531,12 @@ class PARQUET_EXPORT WriterProperties { } /// Used to set minimum compression ratio threshold for V2 page data values - /// compression. If min_space_savings unset, keep compression if compressed value is - /// smaller. - Builder* min_space_savings(std::optional min_space_savings) { - if (min_space_savings && - !(*min_space_savings >= 0.0 && *min_space_savings <= 1.0)) { - throw ParquetException("min_space_savings must be in range [0, 1]"); + /// compression. The default is 0.0: keep compression if values become smaller. + Builder* min_compression_savings(double min_compression_savings) { + if (!(min_compression_savings >= 0.0 && min_compression_savings <= 1.0)) { + throw ParquetException("min_compression_savings must be in range [0, 1]"); } - min_space_savings_ = min_space_savings; + min_compression_savings_ = min_compression_savings; return this; } @@ -896,8 +894,9 @@ class PARQUET_EXPORT WriterProperties { /// \brief Build the WriterProperties with the builder parameters. /// \return The WriterProperties defined by the builder. std::shared_ptr build() { - if (min_space_savings_ && data_page_version_ == ParquetDataPageVersion::V1) { - throw ParquetException("min_space_savings requires V2 data pages"); + if (min_compression_savings_ != 0.0 && + data_page_version_ == ParquetDataPageVersion::V1) { + throw ParquetException("min_compression_savings requires V2 data pages"); } std::unordered_map column_properties; auto get = [&](const std::string& key) -> ColumnProperties& { @@ -934,8 +933,9 @@ class PARQUET_EXPORT WriterProperties { pagesize_, max_rows_per_page_, version_, created_by_, page_checksum_enabled_, size_statistics_level_, std::move(file_encryption_properties_), default_column_properties_, column_properties, data_page_version_, - min_space_savings_, store_decimal_as_integer_, std::move(sorting_columns_), - content_defined_chunking_enabled_, content_defined_chunking_options_)); + min_compression_savings_, store_decimal_as_integer_, + std::move(sorting_columns_), content_defined_chunking_enabled_, + content_defined_chunking_options_)); } private: @@ -949,7 +949,7 @@ class PARQUET_EXPORT WriterProperties { int64_t max_rows_per_page_; ParquetVersion::type version_; ParquetDataPageVersion data_page_version_; - std::optional min_space_savings_ = {}; + double min_compression_savings_ = 0.0; std::string created_by_; bool store_decimal_as_integer_; bool page_checksum_enabled_; @@ -990,7 +990,7 @@ class PARQUET_EXPORT WriterProperties { return parquet_data_page_version_; } - std::optional min_space_savings() const { return min_space_savings_; } + double min_compression_savings() const { return min_compression_savings_; } inline ParquetVersion::type version() const { return parquet_version_; } @@ -1119,7 +1119,7 @@ class PARQUET_EXPORT WriterProperties { std::shared_ptr file_encryption_properties, const ColumnProperties& default_column_properties, const std::unordered_map& column_properties, - ParquetDataPageVersion data_page_version, std::optional min_space_savings, + ParquetDataPageVersion data_page_version, double min_compression_savings, bool store_short_decimal_as_integer, std::vector sorting_columns, bool content_defined_chunking_enabled, CdcOptions content_defined_chunking_options) : pool_(pool), @@ -1129,7 +1129,7 @@ class PARQUET_EXPORT WriterProperties { pagesize_(pagesize), max_rows_per_page_(max_rows_per_page), parquet_data_page_version_(data_page_version), - min_space_savings_(min_space_savings), + min_compression_savings_(min_compression_savings), parquet_version_(version), parquet_created_by_(created_by), store_decimal_as_integer_(store_short_decimal_as_integer), @@ -1149,7 +1149,7 @@ class PARQUET_EXPORT WriterProperties { int64_t pagesize_; int64_t max_rows_per_page_; ParquetDataPageVersion parquet_data_page_version_; - std::optional min_space_savings_; + double min_compression_savings_; ParquetVersion::type parquet_version_; std::string parquet_created_by_; bool store_decimal_as_integer_; diff --git a/cpp/src/parquet/properties_test.cc b/cpp/src/parquet/properties_test.cc index 5758bd3e3b13..b558b144269b 100644 --- a/cpp/src/parquet/properties_test.cc +++ b/cpp/src/parquet/properties_test.cc @@ -80,38 +80,38 @@ TEST(TestWriterProperties, DefaultCompression) { ::arrow::util::kUseDefaultCompressionLevel); } -// checks if WriterProperties correctly store, copy, and clear min_space_savings -TEST(TestWriterProperties, MinSpaceSavings) { +// Checks the default, stored, and copied min_compression_savings. +TEST(TestWriterProperties, MinCompressionSavings) { WriterProperties::Builder builder; - ASSERT_FALSE(builder.build()->min_space_savings().has_value()); + ASSERT_EQ(0.0, builder.build()->min_compression_savings()); builder.data_page_version(ParquetDataPageVersion::V2); for (double savings : {0.0, 0.1, 1.0}) { - auto properties = builder.min_space_savings(savings)->build(); - ASSERT_EQ(std::optional(savings), properties->min_space_savings()); + auto properties = builder.min_compression_savings(savings)->build(); + ASSERT_EQ(savings, properties->min_compression_savings()); auto copied = WriterProperties::Builder(*properties).build(); - ASSERT_EQ(properties->min_space_savings(), copied->min_space_savings()); + ASSERT_EQ(properties->min_compression_savings(), copied->min_compression_savings()); } - auto properties = builder.min_space_savings(std::nullopt)->build(); - ASSERT_FALSE(properties->min_space_savings().has_value()); + auto properties = builder.min_compression_savings(0.0)->build(); + ASSERT_EQ(0.0, properties->min_compression_savings()); } // Tests for invalid settings -TEST(TestWriterProperties, InvalidMinSpaceSavings) { +TEST(TestWriterProperties, InvalidMinCompressionSavings) { for (double savings : {-0.1, 1.1, std::numeric_limits::infinity(), -std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) { - EXPECT_THROW(WriterProperties::Builder().min_space_savings(savings), + EXPECT_THROW(WriterProperties::Builder().min_compression_savings(savings), ParquetException); } } -TEST(TestWriterProperties, MinSpaceSavingsRequiresDataPageV2) { - for (double savings : {0.0, 0.1, 1.0}) { +TEST(TestWriterProperties, MinCompressionSavingsRequiresDataPageV2) { + for (double savings : {0.1, 1.0}) { WriterProperties::Builder builder; - builder.min_space_savings(savings); + builder.min_compression_savings(savings); EXPECT_THROW(builder.build(), ParquetException); builder.data_page_version(ParquetDataPageVersion::V2); @@ -120,7 +120,7 @@ TEST(TestWriterProperties, MinSpaceSavingsRequiresDataPageV2) { builder.data_page_version(ParquetDataPageVersion::V1); EXPECT_THROW(builder.build(), ParquetException); - builder.min_space_savings(std::nullopt); + builder.min_compression_savings(0.0); EXPECT_NO_THROW(builder.build()); } } diff --git a/docs/source/cpp/parquet.rst b/docs/source/cpp/parquet.rst index 58ef0036d889..b1ea78453c7a 100644 --- a/docs/source/cpp/parquet.rst +++ b/docs/source/cpp/parquet.rst @@ -271,12 +271,12 @@ with ``data_page_version``. It's V1 by default; setting to V2 will allow more optimal compression (skipping compressing pages where there isn't a space benefit), but not all readers support this data page version. -For V2 pages, use ``min_space_savings`` to set how much space compression must -save. Use a value from 0.0 to 1.0. For example, ``min_space_savings(0.1)`` requires +For V2 pages, use ``min_compression_savings`` to set how much space compression must +save. Use a value from 0.0 to 1.0. For example, ``min_compression_savings(0.1)`` requires at least 10% savings. If the space savings do not meet the threshold, it is not -compressed. If the ``min_space_savings`` option is unset, compression is kept as -long as compressed values are smaller than original values. Setting a number -with V1 or using a number outside the allowed range throws ``ParquetException``. +compressed. With the default ``0.0``, compression is kept as long as compressed +values are smaller than original values. Setting a nonzero value with V1 or +using a number outside the allowed range throws ``ParquetException``. Compression is off by default, but to get the most out of Parquet, you should also choose a compression codec. You can choose one for the whole file or diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index 4b9d2e2cc8ee..5893fc8924ca 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -664,7 +664,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): store_decimal_as_integer=self._properties["store_decimal_as_integer"], use_content_defined_chunking=self._properties["use_content_defined_chunking"], bloom_filter_options=self._properties["bloom_filter_options"], - min_space_savings=self._properties["min_space_savings"], + min_compression_savings=self._properties["min_compression_savings"], ) def _set_arrow_properties(self): @@ -708,7 +708,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions): use_byte_stream_split=False, column_encoding=None, data_page_version="1.0", - min_space_savings=None, + min_compression_savings=0.0, use_deprecated_int96_timestamps=False, coerce_timestamps=None, allow_truncated_timestamps=False, diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 28a5c676ec98..9cdba15638e6 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -58,7 +58,7 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( store_decimal_as_integer=*, use_content_defined_chunking=*, bloom_filter_options=*, - min_space_savings=* + min_compression_savings=* ) except * diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 9574b9c06b79..0e9b3e38d569 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -2077,7 +2077,7 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( store_decimal_as_integer=False, use_content_defined_chunking=False, bloom_filter_options=None, - min_space_savings=None) except *: + min_compression_savings=0.0) except *: """General writer properties""" cdef: @@ -2086,13 +2086,12 @@ cdef shared_ptr[WriterProperties] _create_writer_properties( CdcOptions cdc_options double savings - if min_space_savings is not None: - savings = min_space_savings - if not (0.0 <= savings <= 1.0): - raise ValueError("min_space_savings must be in range [0, 1]") - if data_page_version in (None, "1.0"): - raise ValueError("min_space_savings requires V2 data pages") - props.min_space_savings(optional[double](savings)) + savings = min_compression_savings + if not (0.0 <= savings <= 1.0): + raise ValueError("min_compression_savings must be in range [0, 1]") + if savings != 0.0 and data_page_version in (None, "1.0"): + raise ValueError("min_compression_savings requires V2 data pages") + props.min_compression_savings(savings) # data_page_version @@ -2416,7 +2415,7 @@ cdef class ParquetWriter(_Weakrefable): use_content_defined_chunking=False, write_time_adjusted_to_utc=False, bloom_filter_options=None, - min_space_savings=None): + min_compression_savings=0.0): cdef: shared_ptr[WriterProperties] properties shared_ptr[ArrowWriterProperties] arrow_properties @@ -2454,7 +2453,7 @@ cdef class ParquetWriter(_Weakrefable): store_decimal_as_integer=store_decimal_as_integer, use_content_defined_chunking=use_content_defined_chunking, bloom_filter_options=bloom_filter_options, - min_space_savings=min_space_savings, + min_compression_savings=min_compression_savings, ) arrow_properties = _create_arrow_writer_properties( use_deprecated_int96_timestamps=use_deprecated_int96_timestamps, diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index ce92aa8c7038..a7ad239feac6 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -485,7 +485,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil: cdef cppclass WriterProperties: cppclass Builder: Builder* data_page_version(ParquetDataPageVersion version) - Builder* min_space_savings(optional[double] min_space_savings) except + + Builder* min_compression_savings(double min_compression_savings) except + Builder* version(ParquetVersion version) Builder* compression(ParquetCompression codec) Builder* compression(const c_string& path, diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 6b1502ef33b0..149ed2c0629e 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -990,14 +990,14 @@ def _sanitize_table(table, new_schema, flavor): balance between deduplication ratio and fragmentation. Use norm_level=1 or norm_level=2 to reach a higher deduplication ratio at the expense of fragmentation. -min_space_savings : float or None, default None +min_compression_savings : float, default 0.0 Minimum space savings required to keep compression for each V2 data page. Use a value from 0.0 to 1.0. For example, 0.1 requires at least 10% savings. Savings are measured on encoded values, excluding repetition and definition levels. Compression is kept only if it makes the values smaller. - If None, no extra minimum savings is required. - Setting a number requires ``data_page_version="2.0"``. Using V1 or a number - outside the allowed range raises ``ValueError``. + The default 0.0 requires no extra minimum savings. + Nonzero values require ``data_page_version="2.0"``. Using a nonzero value + with V1 or a number outside the allowed range raises ``ValueError``. """ _parquet_writer_example_doc = """\ @@ -1098,7 +1098,7 @@ def __init__(self, where, schema, filesystem=None, max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking=False, - min_space_savings=None, + min_compression_savings=0.0, **options): if use_deprecated_int96_timestamps is None: # Use int96 timestamps for Spark @@ -1156,7 +1156,7 @@ def __init__(self, where, schema, filesystem=None, max_rows_per_page=max_rows_per_page, bloom_filter_options=bloom_filter_options, use_content_defined_chunking=use_content_defined_chunking, - min_space_savings=min_space_savings, + min_compression_savings=min_compression_savings, **options) self.is_open = True @@ -2049,7 +2049,7 @@ def write_table(table, where, row_group_size=None, version='2.6', max_rows_per_page=None, bloom_filter_options=None, use_content_defined_chunking=False, - min_space_savings=None, + min_compression_savings=0.0, **kwargs): # Implementor's note: when adding keywords here / updating defaults, also # update it in write_to_dataset and _dataset_parquet.pyx ParquetFileWriteOptions @@ -2085,7 +2085,7 @@ def write_table(table, where, row_group_size=None, version='2.6', max_rows_per_page=max_rows_per_page, bloom_filter_options=bloom_filter_options, use_content_defined_chunking=use_content_defined_chunking, - min_space_savings=min_space_savings, + min_compression_savings=min_compression_savings, **kwargs) as writer: writer.write_table(table, row_group_size=row_group_size) except Exception: diff --git a/python/pyarrow/tests/parquet/test_parquet_writer.py b/python/pyarrow/tests/parquet/test_parquet_writer.py index 0281cd362f2c..14abb2e10fda 100644 --- a/python/pyarrow/tests/parquet/test_parquet_writer.py +++ b/python/pyarrow/tests/parquet/test_parquet_writer.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -import random - import pytest import pyarrow as pa @@ -45,6 +43,7 @@ @pytest.mark.parametrize("page_version, minimum", [ ("1.0", None), + ("1.0", 0.0), ("2.0", None), ("2.0", 0.0), ("2.0", 0.1), @@ -52,7 +51,7 @@ ]) @pytest.mark.parametrize("compression", ["none", "snappy"]) @pytest.mark.parametrize("use_writer", [False, True]) -def test_min_space_savings( +def test_min_compression_savings( minimum, page_version, compression, use_writer): table = pa.table({ "value": pa.array([0.0] * 1000, type=pa.float32()) @@ -62,8 +61,9 @@ def test_min_space_savings( compression=compression, use_dictionary=False, data_page_version=page_version, - min_space_savings=minimum, ) + if minimum is not None: + options["min_compression_savings"] = minimum if use_writer: with pq.ParquetWriter(sink, table.schema, **options) as writer: @@ -85,62 +85,38 @@ def test_min_space_savings( assert column.total_compressed_size < column.total_uncompressed_size -def _v2_column_chunk(table, **options): - sink = pa.BufferOutputStream() - pq.write_table(table, sink, compression="snappy", use_dictionary=False, - data_page_version="2.0", **options) - return pq.ParquetFile(sink.getvalue()).metadata.row_group(0).column(0) - - -def test_min_space_savings_threshold(): - rng = random.Random(42) - values = [rng.randrange(16) for _ in range(1000)] - table = pa.table({"v": pa.array(values, pa.int32())}) - - base = _v2_column_chunk(table) - savings = 1 - base.total_compressed_size / base.total_uncompressed_size - assert 0.1 < savings < 0.9 - - kept = _v2_column_chunk(table, min_space_savings=savings - 0.05) - assert kept.total_compressed_size == base.total_compressed_size - - dropped = _v2_column_chunk(table, min_space_savings=savings + 0.05) - assert dropped.total_compressed_size == dropped.total_uncompressed_size - assert dropped.compression == "SNAPPY" - - # Tests invalid settings @pytest.mark.parametrize( "minimum", [-0.1, 1.1, float("inf"), -float("inf"), float("nan")] ) @pytest.mark.parametrize("use_writer", [False, True]) -def test_invalid_min_space_savings(minimum, use_writer): +def test_invalid_min_compression_savings(minimum, use_writer): table = pa.table({"value": [1.0]}) sink = pa.BufferOutputStream() - with pytest.raises(ValueError, match="min_space_savings"): + with pytest.raises(ValueError, match="min_compression_savings"): if use_writer: pq.ParquetWriter( sink, table.schema, data_page_version="2.0", - min_space_savings=minimum) + min_compression_savings=minimum) else: pq.write_table( table, sink, data_page_version="2.0", - min_space_savings=minimum) + min_compression_savings=minimum) -@pytest.mark.parametrize("minimum", [0.0, 0.1, 1.0]) +@pytest.mark.parametrize("minimum", [0.1, 1.0]) @pytest.mark.parametrize("use_writer", [False, True]) -def test_min_space_savings_rejects_v1(minimum, use_writer): +def test_min_compression_savings_rejects_v1(minimum, use_writer): table = pa.table({"value": [1.0]}) sink = pa.BufferOutputStream() - with pytest.raises(ValueError, match="min_space_savings requires V2"): + with pytest.raises(ValueError, match="min_compression_savings requires V2"): if use_writer: - pq.ParquetWriter(sink, table.schema, min_space_savings=minimum) + pq.ParquetWriter(sink, table.schema, min_compression_savings=minimum) else: - pq.write_table(table, sink, min_space_savings=minimum) + pq.write_table(table, sink, min_compression_savings=minimum) @pytest.mark.pandas diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 589bc08f037c..9aa09eb50f61 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -5950,32 +5950,15 @@ def test_write_dataset_content_defined_chunking(tempdir, cdc): @pytest.mark.parquet -@pytest.mark.parametrize("minimum", [None, 0.0, 0.1, 1.0]) -def test_parquet_min_space_savings(tempdir, minimum): +def test_parquet_min_compression_savings(tempdir): import pyarrow.parquet as pq table = pa.table({ "value": pa.array([0.0] * 1000, type=pa.float32()) }) - pq.write_to_dataset( - table, - tempdir, - compression="snappy", - use_dictionary=False, - data_page_version="2.0", - min_space_savings=minimum, - ) - - result = ds.dataset(tempdir, format="parquet").to_table() - assert result.equals(table) - - -@pytest.mark.parquet -def test_dataset_min_space_savings_changes_size(tempdir): - table = pa.table({"value": [0.0] * 1000}) sizes = [] - for minimum in [None, 1.0]: + for minimum in [0.0, 0.1, 1.0]: directory = tempdir / str(minimum) pq.write_to_dataset( table, @@ -5983,11 +5966,13 @@ def test_dataset_min_space_savings_changes_size(tempdir): compression="snappy", use_dictionary=False, data_page_version="2.0", - min_space_savings=minimum, + min_compression_savings=minimum, ) + result = ds.dataset(directory, format="parquet").to_table() + assert result.equals(table) sizes.append(next(directory.iterdir()).stat().st_size) - assert sizes[1] > sizes[0] + assert sizes[-1] > max(sizes[:-1]) def test_make_write_options_error(): From 714a09795b80738a8fc143f7a430daad66e8d8e7 Mon Sep 17 00:00:00 2001 From: Mashrafee Aryan <121733421+MashrafeeAryan@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:38:22 -0500 Subject: [PATCH 14/15] Update docs/source/cpp/parquet.rst Co-authored-by: Antoine Pitrou --- docs/source/cpp/parquet.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/cpp/parquet.rst b/docs/source/cpp/parquet.rst index b1ea78453c7a..da04389bcc5f 100644 --- a/docs/source/cpp/parquet.rst +++ b/docs/source/cpp/parquet.rst @@ -273,10 +273,10 @@ benefit), but not all readers support this data page version. For V2 pages, use ``min_compression_savings`` to set how much space compression must save. Use a value from 0.0 to 1.0. For example, ``min_compression_savings(0.1)`` requires -at least 10% savings. If the space savings do not meet the threshold, it is not -compressed. With the default ``0.0``, compression is kept as long as compressed -values are smaller than original values. Setting a nonzero value with V1 or -using a number outside the allowed range throws ``ParquetException``. +at least 10% savings. If the space savings for a page do not meet the threshold, +it is not compressed. With the default ``0.0``, compression is kept as long as +compressed values are smaller than original values. Setting a nonzero value with +V1 or using a number outside the allowed range throws ``ParquetException``. Compression is off by default, but to get the most out of Parquet, you should also choose a compression codec. You can choose one for the whole file or From 5dae58c0430ff3acb76256aa6459192abbe773b1 Mon Sep 17 00:00:00 2001 From: Mashrfee Aryan Date: Tue, 27 Oct 2026 10:40:41 -0500 Subject: [PATCH 15/15] Remove extra space in Parquet docs --- docs/source/cpp/parquet.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/cpp/parquet.rst b/docs/source/cpp/parquet.rst index da04389bcc5f..938ef0591be7 100644 --- a/docs/source/cpp/parquet.rst +++ b/docs/source/cpp/parquet.rst @@ -273,7 +273,7 @@ benefit), but not all readers support this data page version. For V2 pages, use ``min_compression_savings`` to set how much space compression must save. Use a value from 0.0 to 1.0. For example, ``min_compression_savings(0.1)`` requires -at least 10% savings. If the space savings for a page do not meet the threshold, +at least 10% savings. If the space savings for a page do not meet the threshold, it is not compressed. With the default ``0.0``, compression is kept as long as compressed values are smaller than original values. Setting a nonzero value with V1 or using a number outside the allowed range throws ``ParquetException``.