From c7617b1926494bb3e771854cd58d9b3177b002b1 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 8 Jul 2026 13:11:23 -0400 Subject: [PATCH] test datafusion PR 23391 --- native/Cargo.lock | 217 ++++++------------ native/Cargo.toml | 42 ++++ .../comet/exec/CometNativeReaderSuite.scala | 86 ++++++- 3 files changed, 194 insertions(+), 151 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 40edf8e5e9..1cc0359d15 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1885,8 +1885,7 @@ dependencies = [ [[package]] name = "datafusion" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-schema", @@ -1934,8 +1933,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "async-trait", @@ -1959,8 +1957,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "async-trait", @@ -2156,8 +2153,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-ipc", @@ -2182,8 +2178,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "futures", "log", @@ -2193,8 +2188,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "async-compression", @@ -2229,8 +2223,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-ipc", @@ -2253,8 +2246,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "async-trait", @@ -2276,8 +2268,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "async-trait", @@ -2299,8 +2290,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "async-trait", @@ -2330,14 +2320,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" [[package]] name = "datafusion-execution" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-buffer", @@ -2359,8 +2347,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-schema", @@ -2381,8 +2368,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2393,8 +2379,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-buffer", @@ -2425,8 +2410,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2446,8 +2430,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2458,8 +2441,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-ord", @@ -2483,8 +2465,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "async-trait", @@ -2499,8 +2480,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2516,8 +2496,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2526,8 +2505,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "datafusion-doc", "quote", @@ -2537,8 +2515,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "chrono", @@ -2556,8 +2533,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2577,8 +2553,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2592,8 +2567,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "chrono", @@ -2609,8 +2583,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2627,8 +2600,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "arrow-data", @@ -2660,8 +2632,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "datafusion-common", @@ -2676,8 +2647,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "async-trait", "datafusion-common", @@ -2690,8 +2660,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "bigdecimal", @@ -2720,8 +2689,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" dependencies = [ "arrow", "bigdecimal", @@ -2967,7 +2935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3857,7 +3825,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5348,7 +5316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5367,7 +5335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -5470,7 +5438,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -5961,7 +5929,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6019,7 +5987,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6597,10 +6565,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7281,7 +7249,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7376,15 +7344,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -7418,30 +7377,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -7454,12 +7396,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -7472,12 +7408,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -7490,24 +7420,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -7520,12 +7438,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -7538,12 +7450,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -7556,12 +7462,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -7574,12 +7474,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "wit-bindgen" version = "0.57.1" @@ -7750,3 +7644,28 @@ dependencies = [ "cc", "pkg-config", ] + +[[patch.unused]] +name = "datafusion-datasource-avro" +version = "54.0.0" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" + +[[patch.unused]] +name = "datafusion-ffi" +version = "54.0.0" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" + +[[patch.unused]] +name = "datafusion-proto" +version = "54.0.0" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" + +[[patch.unused]] +name = "datafusion-proto-common" +version = "54.0.0" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" + +[[patch.unused]] +name = "datafusion-substrait" +version = "54.0.0" +source = "git+https://github.com/mbutrovich/datafusion?branch=comet-4859-nested-projection#61efc9c87183ce8262fea3c8fff718874c8b430d" diff --git a/native/Cargo.toml b/native/Cargo.toml index dda029bba7..8a1a529565 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -78,3 +78,45 @@ codegen-units = 16 # Parallel codegen (faster compile, slightly larger binary) debug-assertions = true panic = "unwind" # Allow panics to be caught and logged across FFI boundary # overflow-checks inherited as false from release + +# TEMPORARY (issue #4859): build against the DataFusion fork branch carrying the +# nested-projection leaf-mask fix. The branch is off tag 54.0.0, so it is API-identical +# to the crates.io 54.0.0 build. Drop this once the fix lands in a DataFusion release. +[patch.crates-io] +datafusion = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-catalog = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-catalog-listing = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-common = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-common-runtime = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-datasource = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-datasource-arrow = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-datasource-avro = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-datasource-csv = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-datasource-json = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-datasource-parquet = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-doc = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-execution = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-expr = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-expr-common = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-ffi = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-functions = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-functions-aggregate = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-functions-aggregate-common = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-functions-nested = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-functions-table = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-functions-window = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-functions-window-common = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-macros = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-optimizer = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-physical-expr = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-physical-expr-adapter = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-physical-expr-common = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-physical-optimizer = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-physical-plan = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-proto = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-proto-common = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-pruning = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-session = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-spark = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-sql = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } +datafusion-substrait = { git = "https://github.com/mbutrovich/datafusion", branch = "comet-4859-nested-projection" } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala index 876565c5e5..35557724c4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala @@ -29,12 +29,12 @@ import org.apache.parquet.hadoop.api.WriteSupport import org.apache.parquet.hadoop.api.WriteSupport.WriteContext import org.apache.parquet.io.api.RecordConsumer import org.apache.parquet.schema.MessageTypeParser -import org.apache.spark.sql.{CometTestBase, Row} +import org.apache.spark.sql.{CometTestBase, DataFrame, Row} import org.apache.spark.sql.comet.CometNativeScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.{array, col} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{IntegerType, LongType, NullType, StringType, StructType} +import org.apache.spark.sql.types._ import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus @@ -794,6 +794,7 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper class Builder extends ParquetWriter.Builder[RecordConsumer => Unit, Builder](new Path(path)) { override def getWriteSupport(conf: Configuration): WriteSupport[RecordConsumer => Unit] = writeSupport + override def self(): Builder = this } val writer = new Builder().build() @@ -877,6 +878,87 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper } } } + test("issue #4859: native scan honors pruned nested ReadSchema for array") { + // Regression repro: the native scan pushes only a top-level projection vector into + // DataFusion, so a plain Column("events") ref yields ProjectionMask::roots and the + // whole nested column (all leaves) is read even though Spark's pruned ReadSchema asks + // for a single leaf. Selecting only the small `id` leaf must scan far fewer bytes than + // also selecting the large `payload` leaf; today they are ~equal. + withTempPath { path => + withSQLConf( + SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "1", + SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key -> "true") { + // events: array>. `payload` is a distinct, + // high-entropy uuid per row so it dominates the on-disk column size and resists + // compression; `id` is a sequential long that compresses to almost nothing. + spark + .range(0, 20000) + .selectExpr("array(named_struct('id', id, 'payload', uuid())) as events") + .repartition(1) + .write + .mode("overwrite") + .parquet(path.toString) + + def bytesScanned(df: DataFrame): Long = { + val (_, cometPlan) = checkSparkAnswerAndOperator(df) + val scans = cometPlan.collect { case n: CometNativeScanExec => n } + assert(scans.nonEmpty, "Expected a CometNativeScanExec") + scans.map(_.metrics("bytes_scanned").value).sum + } + + val idOnly = + bytesScanned(spark.read.parquet(path.toString).selectExpr("events.id")) + val idAndPayload = + bytesScanned( + spark.read.parquet(path.toString).selectExpr("events.id", "events.payload")) + + assert( + idOnly < idAndPayload / 2, + "native scan read the whole nested column despite pruned ReadSchema " + + s"(bytes_scanned: events.id=$idOnly, events.id+payload=$idAndPayload)") + } + } + } + + test("issue #4859: leaf pruning through array> drops siblings at each level") { + withTempPath { path => + withSQLConf( + SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "1", + SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key -> "true") { + // events: array>>. `blob` is the dominant + // high-entropy leaf. Selecting events.inner.a must prune both the top-level + // sibling `id` and the inner sibling `blob`. + spark + .range(0, 20000) + .selectExpr( + "array(named_struct('id', id, 'inner', named_struct('a', id, 'blob', uuid()))) as events") + .repartition(1) + .write + .mode("overwrite") + .parquet(path.toString) + + def bytesScanned(df: DataFrame): Long = { + val (_, cometPlan) = checkSparkAnswerAndOperator(df) + val scans = cometPlan.collect { case n: CometNativeScanExec => n } + assert(scans.nonEmpty, "Expected a CometNativeScanExec") + scans.map(_.metrics("bytes_scanned").value).sum + } + + val aOnly = + bytesScanned(spark.read.parquet(path.toString).selectExpr("events.inner.a")) + val aAndBlob = + bytesScanned( + spark.read + .parquet(path.toString) + .selectExpr("events.inner.a", "events.inner.blob")) + + assert( + aOnly < aAndBlob / 2, + "native scan read unrequested nested leaves through array> " + + s"(bytes_scanned: events.inner.a=$aOnly, events.inner.a+blob=$aAndBlob)") + } + } + } } private class DirectWriteSupport(schema: org.apache.parquet.schema.MessageType)