From 3abc960035ec8962289f4cdd972c91ece59838a4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Jul 2026 11:25:08 -0600 Subject: [PATCH 1/2] chore: WIP upgrade to DataFusion main (63e25c15) + arrow/parquet 59.1.0 --- native/Cargo.lock | 533 ++++++++++++++++++++++++++--------------- native/Cargo.toml | 12 +- native/core/Cargo.toml | 2 +- 3 files changed, 349 insertions(+), 198 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index c54812c818..c43777c53b 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -232,23 +232,23 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", + "arrow-arith 59.1.0", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-cast 59.1.0", "arrow-csv", - "arrow-data", - "arrow-ipc", + "arrow-data 59.1.0", + "arrow-ipc 59.1.0", "arrow-json", - "arrow-ord", + "arrow-ord 59.1.0", "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", + "arrow-string 59.1.0", ] [[package]] @@ -257,10 +257,24 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-arith" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", "chrono", "num-traits", ] @@ -272,9 +286,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" +dependencies = [ + "ahash", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", "chrono", "chrono-tz", "half", @@ -296,18 +328,51 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-buffer" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ord 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-ord 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "atoi", "base64", "chrono", @@ -320,13 +385,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 59.1.0", + "arrow-cast 59.1.0", + "arrow-schema 59.1.0", "chrono", "csv", "csv-core", @@ -339,8 +404,21 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 58.3.0", + "arrow-schema 58.3.0", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +dependencies = [ + "arrow-buffer 59.1.0", + "arrow-schema 59.1.0", "half", "num-integer", "num-traits", @@ -352,11 +430,25 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "flatbuffers", +] + +[[package]] +name = "arrow-ipc" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "flatbuffers", "lz4_flex", "zstd", @@ -364,16 +456,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-cast 59.1.0", + "arrow-ord 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "chrono", "half", "indexmap 2.14.0", @@ -393,23 +485,36 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", +] + +[[package]] +name = "arrow-ord" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", ] [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", "half", ] @@ -418,6 +523,12 @@ name = "arrow-schema" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" + +[[package]] +name = "arrow-schema" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -431,10 +542,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "num-traits", +] + +[[package]] +name = "arrow-select" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" +dependencies = [ + "ahash", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", "num-traits", ] @@ -444,11 +569,28 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "arrow-string" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +dependencies = [ + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "memchr", "num-traits", "regex", @@ -1892,11 +2034,10 @@ dependencies = [ [[package]] name = "datafusion" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-schema", + "arrow-schema 59.1.0", "async-trait", "chrono", "datafusion-catalog", @@ -1926,11 +2067,11 @@ dependencies = [ "datafusion-sql", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", - "parquet", + "parquet 59.1.0", "sqlparser", "tempfile", "tokio", @@ -1941,8 +2082,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "async-trait", @@ -1956,7 +2096,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1966,8 +2106,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "async-trait", @@ -1981,7 +2120,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", ] @@ -2025,7 +2164,7 @@ dependencies = [ "once_cell", "opendal", "parking_lot", - "parquet", + "parquet 59.1.0", "paste", "pprof", "procfs", @@ -2079,7 +2218,7 @@ dependencies = [ "jni 0.22.4", "lazy_static", "once_cell", - "parquet", + "parquet 59.1.0", "paste", "prost", "regex", @@ -2128,7 +2267,7 @@ dependencies = [ "jni 0.21.1", "log", "lz4_flex", - "parquet", + "parquet 59.1.0", "simd-adler32", "snap", "tempfile", @@ -2163,23 +2302,23 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-ipc", - "arrow-schema", + "arrow-ipc 59.1.0", + "arrow-schema 59.1.0", "chrono", "foldhash 0.2.0", "half", "hashbrown 0.17.1", "hex", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", - "parquet", + "parquet 59.1.0", "sqlparser", "tokio", "uuid", @@ -2189,8 +2328,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "futures", "log", @@ -2200,8 +2338,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "async-compression", @@ -2221,7 +2358,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -2236,11 +2373,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-ipc", + "arrow-ipc 59.1.0", "async-trait", "bytes", "datafusion-common", @@ -2252,7 +2388,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] @@ -2260,8 +2396,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "async-trait", @@ -2283,8 +2418,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "async-trait", @@ -2306,10 +2440,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", + "arrow-schema 59.1.0", "async-trait", "bytes", "datafusion-common", @@ -2326,29 +2460,28 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", - "parquet", + "parquet 59.1.0", "tokio", ] [[package]] name = "datafusion-doc" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" [[package]] name = "datafusion-execution" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-buffer", + "arrow-buffer 59.1.0", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -2357,20 +2490,21 @@ dependencies = [ "log", "object_store", "parking_lot", - "parquet", + "parquet 59.1.0", "rand 0.9.4", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-schema", + "arrow-schema 59.1.0", "async-trait", "chrono", "datafusion-common", @@ -2380,7 +2514,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "serde_json", "sqlparser", ] @@ -2388,23 +2522,21 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-buffer", + "arrow-buffer 59.1.0", "base64", "blake2", "blake3", @@ -2418,7 +2550,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -2432,8 +2564,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", @@ -2444,7 +2575,6 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", "log", "num-traits", @@ -2453,8 +2583,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", @@ -2465,11 +2594,10 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-ord", + "arrow-ord 59.1.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2481,7 +2609,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2490,8 +2618,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "async-trait", @@ -2506,8 +2633,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", @@ -2523,8 +2649,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2533,8 +2658,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "datafusion-doc", "quote", @@ -2544,8 +2668,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "chrono", @@ -2554,7 +2677,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "regex", "regex-syntax", @@ -2563,8 +2686,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", @@ -2575,7 +2697,7 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "tokio", @@ -2584,8 +2706,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", @@ -2593,14 +2714,13 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "chrono", @@ -2608,7 +2728,7 @@ dependencies = [ "datafusion-expr-common", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] @@ -2616,8 +2736,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", @@ -2628,21 +2747,21 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-plan" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", + "arrow-data 59.1.0", + "arrow-ipc 59.1.0", + "arrow-ord 59.1.0", + "arrow-schema 59.1.0", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2656,19 +2775,19 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "datafusion-common", @@ -2683,8 +2802,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "async-trait", "datafusion-common", @@ -2697,8 +2815,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "bigdecimal", @@ -2727,8 +2844,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +source = "git+https://github.com/apache/datafusion?rev=63e25c15bc72e731c174ba03d1143b243cc1aaae#63e25c15bc72e731c174ba03d1143b243cc1aaae" dependencies = [ "arrow", "bigdecimal", @@ -3608,14 +3724,14 @@ dependencies = [ "anyhow", "apache-avro", "array-init", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 58.3.0", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-ord 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "arrow-string 58.3.0", "as-any", "async-trait", "backon", @@ -3634,7 +3750,7 @@ dependencies = [ "murmur3", "once_cell", "ordered-float 4.6.0", - "parquet", + "parquet 58.3.0", "rand 0.9.4", "reqwest 0.12.28", "roaring", @@ -4912,12 +5028,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ipc 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "paste", + "ring", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" +dependencies = [ + "ahash", + "arrow-array 59.1.0", + "arrow-buffer 59.1.0", + "arrow-data 59.1.0", + "arrow-ipc 59.1.0", + "arrow-schema 59.1.0", + "arrow-select 59.1.0", "base64", "brotli", "bytes", @@ -4939,7 +5091,6 @@ dependencies = [ "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -4947,12 +5098,12 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c8db065291f088a2aad8ab831853eae1871c0d311c8d0b83bbc3b7e735d0fc" +checksum = "57db04c7f8598c9e12c935a7e953a441009189da3f1d504b63c5dc3346a4d59b" dependencies = [ "arrow", - "arrow-schema", + "arrow-schema 59.1.0", "chrono", "half", "indexmap 2.14.0", @@ -4963,12 +5114,12 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a530e8d5b5e14efcb39c9a6ec55432ad11f6afb7dc4455a79be0dc615fe3cc31" +checksum = "9ecebd5a05d4837104ae401977f5d8ad816045e01b367035c416df3c39540613" dependencies = [ "arrow", - "arrow-schema", + "arrow-schema 59.1.0", "chrono", "half", "indexmap 2.14.0", @@ -4980,11 +5131,11 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ed89908289f67caa2ca078f9ff9aacd6229a313ec92b12bf4f48f613dc2b97" +checksum = "e5fc56a025dc3d5537140fc95d80879750fc1aa4324cf2452f3d1471854c2f8d" dependencies = [ - "arrow-schema", + "arrow-schema 59.1.0", "base64", "chrono", "parquet-variant", diff --git a/native/Cargo.toml b/native/Cargo.toml index e2277596aa..48cc08f7fe 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -34,14 +34,14 @@ edition = "2021" rust-version = "1.88" [workspace.dependencies] -arrow = { version = "58.3.0", features = ["prettyprint", "ffi", "chrono-tz"] } +arrow = { version = "59.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } -parquet = { version = "58.3.0", default-features = false, features = ["experimental"] } -datafusion = { version = "54.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { version = "54.0.0" } -datafusion-physical-expr-adapter = { version = "54.0.0" } -datafusion-spark = { version = "54.0.0", features = ["core"] } +parquet = { version = "59.1.0", default-features = false, features = ["experimental"] } +datafusion = { git = "https://github.com/apache/datafusion", rev = "63e25c15bc72e731c174ba03d1143b243cc1aaae", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { git = "https://github.com/apache/datafusion", rev = "63e25c15bc72e731c174ba03d1143b243cc1aaae" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", rev = "63e25c15bc72e731c174ba03d1143b243cc1aaae" } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "63e25c15bc72e731c174ba03d1143b243cc1aaae", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-common = { path = "common" } datafusion-comet-jni-bridge = { path = "jni-bridge" } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index e657879d33..2b6a53496c 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -91,7 +91,7 @@ jni = { version = "0.22.4", features = ["invocation"] } lazy_static = "1.4" assertables = "10" hex = "0.4.3" -datafusion-functions-nested = { version = "54.0.0" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "63e25c15bc72e731c174ba03d1143b243cc1aaae" } [features] backtrace = ["datafusion/backtrace"] From 8b111043649f551ef93cffffc89a5474b3c9e88d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Jul 2026 12:24:01 -0600 Subject: [PATCH 2/2] fix: adapt native code to DataFusion main and arrow/parquet 59.1.0 Work through the compile breakage surfaced by pinning DataFusion to main: - Drop opt_filter from GroupsAccumulator::merge_batch impls and their delegating call sites (avg, avg_decimal, correlation, covariance, percentile, stddev, sum_decimal, sum_int, variance, merge_as_partial) to match the upstream signature change. - Migrate MutableArrayData::extend/extend_nulls to try_extend/ try_extend_nulls across array_funcs and copy.rs (arrow 59 returns a Result instead of panicking). - Rewire shuffle spill to the new DiskManager API, where create_tmp_file returns Arc and path() returns Option<&Path>. - Replace deprecated TableSchema::from_file_schema/with_table_partition_cols with the TableSchema builder. - Use FixedSizeBinaryArray::try_from (arrow 59 dropped the From impl). The native iceberg scan remains broken: iceberg-rust still depends on arrow 58, so its RecordBatch does not unify with Comet's arrow 59 type. This is blocked on iceberg-rust upgrading to arrow 59. --- native/core/src/execution/columnar_to_row.rs | 13 ++++++---- native/core/src/execution/merge_as_partial.rs | 7 +++--- native/core/src/execution/operators/copy.rs | 8 +++++-- native/core/src/parquet/parquet_exec.rs | 11 +++++---- native/shuffle/src/writers/local/spill.rs | 13 ++++++---- native/spark-expr/src/agg_funcs/avg.rs | 1 - .../spark-expr/src/agg_funcs/avg_decimal.rs | 1 - .../spark-expr/src/agg_funcs/correlation.rs | 7 +++--- native/spark-expr/src/agg_funcs/covariance.rs | 5 ++-- native/spark-expr/src/agg_funcs/percentile.rs | 1 - native/spark-expr/src/agg_funcs/stddev.rs | 3 +-- .../spark-expr/src/agg_funcs/sum_decimal.rs | 6 ----- native/spark-expr/src/agg_funcs/sum_int.rs | 18 -------------- native/spark-expr/src/agg_funcs/variance.rs | 5 ++-- .../src/array_funcs/array_compact.rs | 2 +- .../src/array_funcs/array_insert.rs | 24 +++++++++---------- .../spark-expr/src/array_funcs/array_slice.rs | 2 +- .../spark-expr/src/array_funcs/arrays_zip.rs | 6 ++--- .../src/array_funcs/list_extract.rs | 8 +++---- 19 files changed, 61 insertions(+), 80 deletions(-) diff --git a/native/core/src/execution/columnar_to_row.rs b/native/core/src/execution/columnar_to_row.rs index 14e115cba0..06f5bbd78d 100644 --- a/native/core/src/execution/columnar_to_row.rs +++ b/native/core/src/execution/columnar_to_row.rs @@ -2499,11 +2499,14 @@ mod tests { let schema = vec![DataType::FixedSizeBinary(3)]; let mut ctx = ColumnarToRowContext::new(schema, 100); - let array: ArrayRef = Arc::new(FixedSizeBinaryArray::from(vec![ - Some(&[1u8, 2, 3][..]), - Some(&[4u8, 5, 6][..]), - None, // Test null handling - ])); + let array: ArrayRef = Arc::new( + FixedSizeBinaryArray::try_from(vec![ + Some(&[1u8, 2, 3][..]), + Some(&[4u8, 5, 6][..]), + None, // Test null handling + ]) + .unwrap(), + ); let arrays = vec![array]; let (ptr, offsets, lengths) = ctx.convert(&arrays, 3).unwrap(); diff --git a/native/core/src/execution/merge_as_partial.rs b/native/core/src/execution/merge_as_partial.rs index 5ea26115bf..4608b6c339 100644 --- a/native/core/src/execution/merge_as_partial.rs +++ b/native/core/src/execution/merge_as_partial.rs @@ -207,23 +207,22 @@ impl GroupsAccumulator for MergeAsPartialGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, + _opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { // Redirect update to merge — this is the key trick. self.inner - .merge_batch(values, group_indices, opt_filter, total_num_groups) + .merge_batch(values, group_indices, total_num_groups) } fn merge_batch( &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { self.inner - .merge_batch(values, group_indices, opt_filter, total_num_groups) + .merge_batch(values, group_indices, total_num_groups) } fn evaluate(&mut self, emit_to: EmitTo) -> Result { diff --git a/native/core/src/execution/operators/copy.rs b/native/core/src/execution/operators/copy.rs index 193c385c3c..ca0ae6ddfc 100644 --- a/native/core/src/execution/operators/copy.rs +++ b/native/core/src/execution/operators/copy.rs @@ -37,7 +37,9 @@ pub(crate) fn copy_array(array: &dyn Array) -> ArrayRef { let mut mutable = MutableArrayData::new(vec![&data], false, capacity); - mutable.extend(0, 0, capacity); + mutable + .try_extend(0, 0, capacity) + .expect("copy_array: extend within existing array cannot overflow"); if matches!(array.data_type(), DataType::Dictionary(_, _)) { let copied_dict = make_array(mutable.freeze()); @@ -50,7 +52,9 @@ pub(crate) fn copy_array(array: &dyn Array) -> ArrayRef { let data = values.to_data(); let mut mutable = MutableArrayData::new(vec![&data], false, values.len()); - mutable.extend(0, 0, values.len()); + mutable + .try_extend(0, 0, values.len()) + .expect("copy_array: extend within existing array cannot overflow"); let copied_dict = ref_copied_dict.with_values(make_array(mutable.freeze())); Arc::new(copied_dict) diff --git a/native/core/src/parquet/parquet_exec.rs b/native/core/src/parquet/parquet_exec.rs index d8704d80c0..b64bbee7c1 100644 --- a/native/core/src/parquet/parquet_exec.rs +++ b/native/core/src/parquet/parquet_exec.rs @@ -19,7 +19,7 @@ use crate::execution::operators::ExecutionError; use crate::parquet::encryption_support::{CometEncryptionConfig, ENCRYPTION_FACTORY_ID}; use crate::parquet::parquet_support::SparkParquetOptions; use crate::parquet::schema_adapter::SparkPhysicalExprAdapterFactory; -use arrow::datatypes::{Field, SchemaRef}; +use arrow::datatypes::{Field, FieldRef, SchemaRef}; use datafusion::config::TableParquetOptions; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::parquet::CachedParquetFileReaderFactory; @@ -122,13 +122,14 @@ pub(crate) fn init_datasource_exec( } _ => (Arc::clone(&required_schema), None), }; - let partition_fields: Vec<_> = partition_schema + let partition_fields: Vec = partition_schema .iter() .flat_map(|s| s.fields().iter()) - .map(|f| Arc::new(Field::new(f.name(), f.data_type().clone(), f.is_nullable())) as _) + .map(|f| Arc::new(Field::new(f.name(), f.data_type().clone(), f.is_nullable()))) .collect(); - let table_schema = - TableSchema::from_file_schema(base_schema).with_table_partition_cols(partition_fields); + let table_schema = TableSchema::builder(base_schema) + .with_table_partition_cols(partition_fields) + .build(); let mut parquet_source = ParquetSource::new(table_schema) .with_table_parquet_options(table_parquet_options) diff --git a/native/shuffle/src/writers/local/spill.rs b/native/shuffle/src/writers/local/spill.rs index 450b617421..e12b68a005 100644 --- a/native/shuffle/src/writers/local/spill.rs +++ b/native/shuffle/src/writers/local/spill.rs @@ -20,12 +20,13 @@ use crate::writers::BufBatchWriter; use crate::ShuffleBlockWriter; use arrow::record_batch::RecordBatch; use datafusion::common::DataFusionError; -use datafusion::execution::disk_manager::RefCountedTempFile; use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::execution::SpillFile as DfSpillFile; use std::fs::{File, OpenOptions}; +use std::sync::Arc; struct SpillFile { - temp_file: RefCountedTempFile, + temp_file: Arc, file: File, } @@ -97,7 +98,11 @@ impl SpillWriter { .write(true) .create(true) .truncate(true) - .open(spill_file.path()) + .open(spill_file.path().ok_or_else(|| { + DataFusionError::Execution( + "Spill file backend does not expose a local path".to_string(), + ) + })?) .map_err(|e| { DataFusionError::Execution(format!("Error occurred while spilling {e}")) })?; @@ -112,7 +117,7 @@ impl SpillWriter { pub(crate) fn path(&self) -> Option<&std::path::Path> { self.spill_file .as_ref() - .map(|spill_file| spill_file.temp_file.path()) + .and_then(|spill_file| spill_file.temp_file.path()) } #[cfg(test)] diff --git a/native/spark-expr/src/agg_funcs/avg.rs b/native/spark-expr/src/agg_funcs/avg.rs index 24a9a30991..1af58193aa 100644 --- a/native/spark-expr/src/agg_funcs/avg.rs +++ b/native/spark-expr/src/agg_funcs/avg.rs @@ -283,7 +283,6 @@ where &mut self, values: &[ArrayRef], group_indices: &[usize], - _opt_filter: Option<&arrow::array::BooleanArray>, total_num_groups: usize, ) -> Result<()> { assert_eq!(values.len(), 2, "two arguments to merge_batch"); diff --git a/native/spark-expr/src/agg_funcs/avg_decimal.rs b/native/spark-expr/src/agg_funcs/avg_decimal.rs index a71bbe44ea..6b9d0b37da 100644 --- a/native/spark-expr/src/agg_funcs/avg_decimal.rs +++ b/native/spark-expr/src/agg_funcs/avg_decimal.rs @@ -536,7 +536,6 @@ impl GroupsAccumulator for AvgDecimalGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - _opt_filter: Option<&arrow::array::BooleanArray>, total_num_groups: usize, ) -> Result<()> { assert_eq!(values.len(), 2, "two arguments to merge_batch"); diff --git a/native/spark-expr/src/agg_funcs/correlation.rs b/native/spark-expr/src/agg_funcs/correlation.rs index 189b76c863..469868d978 100644 --- a/native/spark-expr/src/agg_funcs/correlation.rs +++ b/native/spark-expr/src/agg_funcs/correlation.rs @@ -335,7 +335,6 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { assert_eq!(values.len(), 6, "six state columns to merge_batch"); @@ -358,11 +357,11 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator { ]; self.covar - .merge_batch(&covar_state, group_indices, opt_filter, total_num_groups)?; + .merge_batch(&covar_state, group_indices, total_num_groups)?; self.var1 - .merge_batch(&var1_state, group_indices, opt_filter, total_num_groups)?; + .merge_batch(&var1_state, group_indices, total_num_groups)?; self.var2 - .merge_batch(&var2_state, group_indices, opt_filter, total_num_groups)?; + .merge_batch(&var2_state, group_indices, total_num_groups)?; Ok(()) } diff --git a/native/spark-expr/src/agg_funcs/covariance.rs b/native/spark-expr/src/agg_funcs/covariance.rs index e5b77ad33b..4883dc2dd3 100644 --- a/native/spark-expr/src/agg_funcs/covariance.rs +++ b/native/spark-expr/src/agg_funcs/covariance.rs @@ -414,7 +414,6 @@ impl GroupsAccumulator for CovarianceGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - _opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { assert_eq!(values.len(), 4, "four arguments to merge_batch"); @@ -563,8 +562,8 @@ mod groups_tests { let rstate = right.state(EmitTo::All).unwrap(); let mut merged = pop(); - merged.merge_batch(&lstate, &[0], None, 1).unwrap(); - merged.merge_batch(&rstate, &[0], None, 1).unwrap(); + merged.merge_batch(&lstate, &[0], 1).unwrap(); + merged.merge_batch(&rstate, &[0], 1).unwrap(); let merged_result = evaluate(&mut merged)[0].unwrap(); assert!((single - merged_result).abs() < 1e-12); diff --git a/native/spark-expr/src/agg_funcs/percentile.rs b/native/spark-expr/src/agg_funcs/percentile.rs index 3fddfdcb68..c38871804c 100644 --- a/native/spark-expr/src/agg_funcs/percentile.rs +++ b/native/spark-expr/src/agg_funcs/percentile.rs @@ -221,7 +221,6 @@ impl GroupsAccumulator for SparkPercentileGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - _opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { let input_group_values = values[0].as_list::(); diff --git a/native/spark-expr/src/agg_funcs/stddev.rs b/native/spark-expr/src/agg_funcs/stddev.rs index 3389a86af6..fa55649881 100644 --- a/native/spark-expr/src/agg_funcs/stddev.rs +++ b/native/spark-expr/src/agg_funcs/stddev.rs @@ -231,11 +231,10 @@ impl GroupsAccumulator for StddevGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { self.inner - .merge_batch(values, group_indices, opt_filter, total_num_groups) + .merge_batch(values, group_indices, total_num_groups) } fn evaluate(&mut self, emit_to: EmitTo) -> Result { diff --git a/native/spark-expr/src/agg_funcs/sum_decimal.rs b/native/spark-expr/src/agg_funcs/sum_decimal.rs index 4ddf13c5d6..af27728a69 100644 --- a/native/spark-expr/src/agg_funcs/sum_decimal.rs +++ b/native/spark-expr/src/agg_funcs/sum_decimal.rs @@ -541,14 +541,8 @@ impl GroupsAccumulator for SumDecimalGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> DFResult<()> { - debug_assert!( - opt_filter.is_none(), - "opt_filter is not supported in merge_batch" - ); - self.resize_helper(total_num_groups); // For decimal sum, always expect 2 arrays regardless of eval_mode diff --git a/native/spark-expr/src/agg_funcs/sum_int.rs b/native/spark-expr/src/agg_funcs/sum_int.rs index 59a4371772..53585ea6de 100644 --- a/native/spark-expr/src/agg_funcs/sum_int.rs +++ b/native/spark-expr/src/agg_funcs/sum_int.rs @@ -495,14 +495,8 @@ impl GroupsAccumulator for SumIntGroupsAccumulatorLegacy { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> DFResult<()> { - debug_assert!( - opt_filter.is_none(), - "opt_filter is not supported in merge_batch" - ); - if values.len() != 1 { return Err(DataFusionError::Internal(format!( "Invalid state while merging batch. Expected 1 element but found {}", @@ -642,14 +636,8 @@ impl GroupsAccumulator for SumIntGroupsAccumulatorAnsi { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> DFResult<()> { - debug_assert!( - opt_filter.is_none(), - "opt_filter is not supported in merge_batch" - ); - if values.len() != 1 { return Err(DataFusionError::Internal(format!( "Invalid state while merging batch. Expected 1 element but found {}", @@ -826,14 +814,8 @@ impl GroupsAccumulator for SumIntGroupsAccumulatorTry { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> DFResult<()> { - debug_assert!( - opt_filter.is_none(), - "opt_filter is not supported in merge_batch" - ); - if values.len() != 2 { return Err(DataFusionError::Internal(format!( "Invalid state while merging batch. Expected 2 elements but found {}", diff --git a/native/spark-expr/src/agg_funcs/variance.rs b/native/spark-expr/src/agg_funcs/variance.rs index f1841729cf..c3cf2555ae 100644 --- a/native/spark-expr/src/agg_funcs/variance.rs +++ b/native/spark-expr/src/agg_funcs/variance.rs @@ -340,7 +340,6 @@ impl GroupsAccumulator for VarianceGroupsAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - _opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { assert_eq!(values.len(), 3, "three arguments to merge_batch"); @@ -512,8 +511,8 @@ mod groups_tests { let right_state = right.state(EmitTo::All).unwrap(); let mut merged = pop_acc(); - merged.merge_batch(&left_state, &[0], None, 1).unwrap(); - merged.merge_batch(&right_state, &[0], None, 1).unwrap(); + merged.merge_batch(&left_state, &[0], 1).unwrap(); + merged.merge_batch(&right_state, &[0], 1).unwrap(); let merged_result = evaluate(&mut merged)[0].unwrap(); assert!((single_result - merged_result).abs() < 1e-12); diff --git a/native/spark-expr/src/array_funcs/array_compact.rs b/native/spark-expr/src/array_funcs/array_compact.rs index 18e28b9afa..497f17d298 100644 --- a/native/spark-expr/src/array_funcs/array_compact.rs +++ b/native/spark-expr/src/array_funcs/array_compact.rs @@ -146,7 +146,7 @@ fn compact_list( for i in start..end { let is_null = value_nulls.as_ref().map(|n| n.is_null(i)).unwrap_or(false); if !is_null { - mutable.extend(0, i, i + 1); + mutable.try_extend(0, i, i + 1)?; copied += 1; } } diff --git a/native/spark-expr/src/array_funcs/array_insert.rs b/native/spark-expr/src/array_funcs/array_insert.rs index e056c108e0..16954f1243 100644 --- a/native/spark-expr/src/array_funcs/array_insert.rs +++ b/native/spark-expr/src/array_funcs/array_insert.rs @@ -266,17 +266,17 @@ fn array_insert( if pos1 <= len + 1 { // In-range insertion (including appending to end) let corrected = pos1 - 1; // 0-based insertion point - mutable_values.extend(0, start, start + corrected); - mutable_values.extend(1, row_index, row_index + 1); - mutable_values.extend(0, start + corrected, end); + mutable_values.try_extend(0, start, start + corrected)?; + mutable_values.try_extend(1, row_index, row_index + 1)?; + mutable_values.try_extend(0, start + corrected, end)?; final_len = len + 1; } else { // Beyond end: pad with nulls then insert let corrected = pos1 - 1; let padding = corrected - len; - mutable_values.extend(0, start, end); - mutable_values.extend_nulls(padding); - mutable_values.extend(1, row_index, row_index + 1); + mutable_values.try_extend(0, start, end)?; + mutable_values.try_extend_nulls(padding)?; + mutable_values.try_extend(1, row_index, row_index + 1)?; final_len = corrected + 1; // equals pos1 } } else { @@ -289,9 +289,9 @@ fn array_insert( // Legacy: -1 behaves like insert before the last element (corrected = len - k) let base_offset = if legacy_mode { 0 } else { 1 }; let corrected = len - k + base_offset; - mutable_values.extend(0, start, start + corrected); - mutable_values.extend(1, row_index, row_index + 1); - mutable_values.extend(0, start + corrected, end); + mutable_values.try_extend(0, start, start + corrected)?; + mutable_values.try_extend(1, row_index, row_index + 1)?; + mutable_values.try_extend(0, start + corrected, end)?; final_len = len + 1; } else { // Negative index beyond the start (Spark-specific behavior): @@ -300,9 +300,9 @@ fn array_insert( let base_offset = if legacy_mode { 1 } else { 0 }; let target_len = k + base_offset; let padding = target_len.saturating_sub(len + 1); - mutable_values.extend(1, row_index, row_index + 1); // insert item first - mutable_values.extend_nulls(padding); // pad nulls - mutable_values.extend(0, start, end); // append original values + mutable_values.try_extend(1, row_index, row_index + 1)?; // insert item first + mutable_values.try_extend_nulls(padding)?; // pad nulls + mutable_values.try_extend(0, start, end)?; // append original values final_len = target_len; } } diff --git a/native/spark-expr/src/array_funcs/array_slice.rs b/native/spark-expr/src/array_funcs/array_slice.rs index e9dc05330b..fcd3b467a6 100644 --- a/native/spark-expr/src/array_funcs/array_slice.rs +++ b/native/spark-expr/src/array_funcs/array_slice.rs @@ -163,7 +163,7 @@ fn slice_list( } else { let take = std::cmp::min(length_value, arr_len - zero_based_start) as usize; let begin = row_start + zero_based_start as usize; - mutable.extend(0, begin, begin + take); + mutable.try_extend(0, begin, begin + take)?; take }; diff --git a/native/spark-expr/src/array_funcs/arrays_zip.rs b/native/spark-expr/src/array_funcs/arrays_zip.rs index 6f0332d3d8..de4d74bd23 100644 --- a/native/spark-expr/src/array_funcs/arrays_zip.rs +++ b/native/spark-expr/src/array_funcs/arrays_zip.rs @@ -276,15 +276,15 @@ pub fn arrays_zip_inner(args: &[ArrayRef], names: Vec) -> Result { // Null list entry or None (Null-typed) arg — all nulls. if let Some(builder) = builders[col_idx].as_mut() { - builder.extend_nulls(max_len); + builder.try_extend_nulls(max_len)?; } } } diff --git a/native/spark-expr/src/array_funcs/list_extract.rs b/native/spark-expr/src/array_funcs/list_extract.rs index d68784ca70..0944abb987 100644 --- a/native/spark-expr/src/array_funcs/list_extract.rs +++ b/native/spark-expr/src/array_funcs/list_extract.rs @@ -287,10 +287,10 @@ fn list_extract( let len = offset_window[1].as_usize() - start; if list_array.is_null(row) { - mutable.extend_nulls(1); + mutable.try_extend_nulls(1)?; } else if let Some(index) = index { if let Some(i) = adjust_index(index, len)? { - mutable.extend(0, start + i, start + i + 1); + mutable.try_extend(0, start + i, start + i + 1)?; } else if fail_on_error { // Throw appropriate error based on whether this is element_at (one_based=true) // or GetArrayItem (one_based=false) @@ -309,11 +309,11 @@ fn list_extract( }; return Err(error_wrapper(error)); } else { - mutable.extend(1, 0, 1); + mutable.try_extend(1, 0, 1)?; } } else { // index is NULL → result is NULL - mutable.extend_nulls(1); + mutable.try_extend_nulls(1)?; } }