diff --git a/benchmarks/avro-rust/Cargo.lock b/benchmarks/avro-rust/Cargo.lock new file mode 100644 index 0000000..a8800ae --- /dev/null +++ b/benchmarks/avro-rust/Cargo.lock @@ -0,0 +1,1185 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "digest", + "log", + "miniz_oxide", + "num-bigint 0.4.8", + "ouroboros", + "quad-rand", + "rand", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum", + "thiserror", + "uuid", +] + +[[package]] +name = "arrow-array" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-avro" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edad137a3609e2c6a4af0b5ff1c5c049b7af532649d235c7e22e9a9cad933b35" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "bzip2", + "crc", + "flate2", + "indexmap", + "liblzma", + "rand", + "serde", + "serde_json", + "snap", + "strum_macros", + "uuid", + "zstd", +] + +[[package]] +name = "arrow-buffer" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-schema" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "avro_rust" +version = "0.1.0" +dependencies = [ + "apache-avro", + "arrow-array", + "arrow-avro", + "flate2", + "serde_json", +] + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "liblzma" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zlib-rs" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/benchmarks/avro-rust/Cargo.toml b/benchmarks/avro-rust/Cargo.toml new file mode 100644 index 0000000..186720c --- /dev/null +++ b/benchmarks/avro-rust/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "avro_rust" +version = "0.1.0" +edition = "2024" +publish = false + +# Rust Avro readers for measure_avro_rust.py: arrow-avro (Apache Arrow's columnar +# reader) and apache-avro (the Apache Avro project's Rust crate, one Value per +# record). Built twice by the measure script: default features (arrow-avro's own +# deflate backend, miniz_oxide) and `--features zlib-rs` (flate2 switched to +# zlib-rs, the fastest pure-Rust inflater) so arrow-avro gets its best deflate. + +[dependencies] +apache-avro = "0.22.0" +arrow-array = "60.0.0" +arrow-avro = "60.0.0" +flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } +serde_json = "1" + +[features] +zlib-rs = ["flate2/zlib-rs"] + +[profile.release] +lto = "fat" +codegen-units = 1 diff --git a/benchmarks/avro-rust/examples/inspect.rs b/benchmarks/avro-rust/examples/inspect.rs new file mode 100644 index 0000000..c40ffef --- /dev/null +++ b/benchmarks/avro-rust/examples/inspect.rs @@ -0,0 +1,18 @@ +//! Print arrow-avro's output schema and row 0 of the first batch, for checking +//! that it decodes gen_avro.py's file to the same values as rugo/fastavro. +//! +//! cargo run --release --example inspect -- + +use std::io::Cursor; + +fn main() { + let path = std::env::args().nth(1).expect("usage: inspect "); + let data = std::fs::read(path).expect("read file"); + let mut r = arrow_avro::reader::ReaderBuilder::new() + .build(Cursor::new(&data[..])) + .expect("arrow-avro reader"); + let batch = r.next().expect("a batch").expect("decode"); + for (field, col) in batch.schema().fields().iter().zip(batch.columns()) { + println!("{} {:?}\n {:?}", field.name(), field.data_type(), col.slice(0, 1)); + } +} diff --git a/benchmarks/avro-rust/src/main.rs b/benchmarks/avro-rust/src/main.rs new file mode 100644 index 0000000..e5fa139 --- /dev/null +++ b/benchmarks/avro-rust/src/main.rs @@ -0,0 +1,108 @@ +//! Run one (reader, shape) Avro measurement - the Rust counterpart of avro_one.py. +//! +//! avro_rust [batch_size] [utf8_view] +//! +//! Prints one JSON line: best/median wall time in ms and the row count. Called by +//! measure_avro_rust.py; not intended for direct use. +//! +//! Every reader decodes the same in-memory bytes; reading the file is outside the +//! timing. One thread throughout (neither crate parallelises a single file). +//! +//! arrow-avro arrow_avro::reader::Reader -> Arrow RecordBatches (columnar; the +//! counterpart of rugo's Morsels). batch_size defaults to the +//! crate's 1024; utf8_view=1 decodes strings to StringViewArray. +//! apache-avro apache_avro::Reader -> one Value per record (row-oriented) +//! +//! Shapes: `all` reads every field; `narrow` reads id, amount and country. +//! arrow-avro is given a projection (pruned reader schema); apache-avro is given a +//! reader schema of just those fields, which is how it skips the rest. + +use std::io::Cursor; +use std::time::Instant; + +const NARROW: [&str; 3] = ["id", "amount", "country"]; + +fn writer_schema_json(data: &[u8]) -> serde_json::Value { + let reader = apache_avro::Reader::new(Cursor::new(data)).expect("open avro"); + serde_json::from_str(&reader.writer_schema().canonical_form()).expect("schema json") +} + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() < 5 { + eprintln!("usage: avro_rust [batch_size] [utf8_view]"); + std::process::exit(2); + } + let (reader, shape, path) = (args[1].as_str(), args[2].as_str(), &args[3]); + let reps: usize = args[4].parse().expect("reps"); + let batch_size: usize = args.get(5).map(|s| s.parse().expect("batch_size")).unwrap_or(1024); + let utf8_view = args.get(6).map(|s| s == "1").unwrap_or(false); + let narrow = match shape { + "all" => false, + "narrow" => true, + _ => panic!("unknown shape {shape}"), + }; + let data = std::fs::read(path).expect("read file"); + + // Top-level field indexes / pruned schema for the narrow shape. + let schema = writer_schema_json(&data); + let fields = schema["fields"].as_array().expect("record fields"); + let projection: Vec = NARROW + .iter() + .map(|n| fields.iter().position(|f| f["name"] == *n).expect("narrow field")) + .collect(); + let mut pruned = schema.clone(); + pruned["fields"] = serde_json::Value::Array(projection.iter().map(|&i| fields[i].clone()).collect()); + let apache_reader_schema = apache_avro::Schema::parse_str(&pruned.to_string()).expect("reader schema"); + + let go = || -> usize { + match reader { + "arrow-avro" => { + let mut b = arrow_avro::reader::ReaderBuilder::new() + .with_batch_size(batch_size) + .with_utf8_view(utf8_view); + if narrow { + b = b.with_projection(projection.clone()); + } + let r = b.build(Cursor::new(&data[..])).expect("arrow-avro reader"); + r.map(|batch| batch.expect("arrow-avro batch").num_rows()).sum() + } + "apache-avro" => { + let r = if narrow { + apache_avro::Reader::builder(Cursor::new(&data[..])) + .reader_schema(&apache_reader_schema) + .build() + } else { + apache_avro::Reader::new(Cursor::new(&data[..])) + } + .expect("apache-avro reader"); + r.map(|v| { + v.expect("apache-avro record"); + 1usize + }) + .sum() + } + _ => panic!("unknown reader {reader}"), + } + }; + + let mut times = Vec::with_capacity(reps); + let mut rows = 0; + for _ in 0..reps { + let t = Instant::now(); + rows = std::hint::black_box(go()); + times.push(t.elapsed().as_secs_f64() * 1000.0); + } + let best = times.iter().cloned().fold(f64::INFINITY, f64::min); + let mut sorted = times.clone(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let median = if reps % 2 == 1 { + sorted[reps / 2] + } else { + (sorted[reps / 2 - 1] + sorted[reps / 2]) / 2.0 + }; + println!( + "{{\"best_ms\": {:.1}, \"median_ms\": {:.1}, \"rows\": {}}}", + best, median, rows + ); +} diff --git a/benchmarks/avro_one.py b/benchmarks/avro_one.py index e70bded..56ed425 100644 --- a/benchmarks/avro_one.py +++ b/benchmarks/avro_one.py @@ -9,6 +9,7 @@ timing. rugo rugo.avro.read_avro -> columnar Morsels (no Python object per value) + rugo-1t the same, max_threads=1 rugo+pylist the same, then every column .to_pylist() - the cost of ending in Python objects, which is what the other two return fastavro fastavro.reader -> one dict per record @@ -32,14 +33,15 @@ def narrow_schema(): return {**schema, "fields": [f for f in schema["fields"] if f["name"] in NARROW]} -if reader in ("rugo", "rugo+pylist"): +if reader in ("rugo", "rugo-1t", "rugo+pylist"): from rugo.avro import read_avro columns = NARROW if shape == "narrow" else None to_python = reader == "rugo+pylist" + threads = 1 if reader == "rugo-1t" else 0 def go(): n = 0 - with read_avro(data, columns=columns) as r: + with read_avro(data, columns=columns, max_threads=threads) as r: for morsel in r: n += morsel.num_rows if to_python: diff --git a/benchmarks/measure_avro_rust.py b/benchmarks/measure_avro_rust.py new file mode 100644 index 0000000..5a7418f --- /dev/null +++ b/benchmarks/measure_avro_rust.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Race rugo's Avro reader against the Rust Avro readers on this machine. + + python3 benchmarks/measure_avro_rust.py [--rows 200000] [--rounds 5] [--reps 5] + [--python 3.14] [--rugo-src /path/to/opteryx-core] + +Needs `uv` and `cargo` on PATH. Same file and method as measure_avro.py: gen_avro.py +writes a deterministic 20-field file with fastavro (so no reader is timed on its own +writer's output), each reader decodes the same in-memory bytes in a fresh process, +reading the file is outside the timing. Both codecs, `deflate` and `null` (`null` +isolates decode from inflate). + +Arms - single thread unless the name says otherwise: + rugo-1t PyPI rugo, read_avro(max_threads=1) -> Morsels (avro_one.py) + rugo PyPI rugo, every core (rugo's default) - NOT part of the race + rugo-src-1t / -src the same from an opteryx-core source tree (--rugo-src) + arrow-avro Apache Arrow's arrow-avro -> RecordBatches, crate defaults + (batch 1024, StringArray, miniz_oxide inflate) + arrow-avro-tuned the same with batch 65,536 (rugo's morsel size) and flate2 on + zlib-rs, the fastest Rust inflater - arrow-avro at its best + arrow-avro-tuned-view tuned + StringViewArray strings + apache-avro the Apache Avro Rust crate -> one Value per record + +Rounds interleave the arms (order rotated each round) so thermal drift and order +bias land on every arm; each (round, arm) is a fresh process running `reps` reads. +Per arm: best = min over every read, median = median of the per-process medians. +Writes benchmarks/results/-avro-rust.json. Results are only comparable to +earlier runs from the same machine. +""" +import argparse, datetime, json, os, platform, statistics, subprocess, tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +CRATE = f"{HERE}/avro-rust" +ap = argparse.ArgumentParser() +ap.add_argument("--rows", type=int, default=200_000) +ap.add_argument("--rounds", type=int, default=5) +ap.add_argument("--reps", type=int, default=5) +ap.add_argument("--python", default="3.14") +ap.add_argument("--rugo-src") +ap.add_argument("--workdir", default=tempfile.mkdtemp(prefix="rugo-avro-rust-")) +a = ap.parse_args() +W = a.workdir + + +def sh(*cmd, env=None): + return subprocess.run(cmd, check=True, capture_output=True, text=True, env=env).stdout + + +def venv(name, pkgs): + p = f"{W}/venv-{name}" + sh("uv", "venv", "--python", a.python, p) + sh("uv", "pip", "install", "--python", f"{p}/bin/python", *pkgs) + return f"{p}/bin/python" + + +# --- builds: one binary per flate2 backend +print("building Rust readers", flush=True) +BIN = {} +for variant, extra in (("default", []), ("zlib-rs", ["--features", "zlib-rs"])): + sh("cargo", "build", "--release", "--quiet", "--manifest-path", f"{CRATE}/Cargo.toml", + "--target-dir", f"{CRATE}/target/{variant}", *extra) + BIN[variant] = f"{CRATE}/target/{variant}/release/avro_rust" +lock = open(f"{CRATE}/Cargo.lock").read().split("[[package]]") +crate_versions = {} +for block in lock: + fields = dict(line.split(" = ", 1) for line in block.strip().splitlines() if " = " in line) + name = fields.get("name", "").strip('"') + if name in ("arrow-avro", "apache-avro", "flate2", "zlib-rs", "miniz_oxide"): + crate_versions[name] = fields["version"].strip('"') + +# --- Python side +pypi = venv("pypi", ["rugo", "fastavro"]) +ARMS = {"rugo-1t": [pypi, f"{HERE}/avro_one.py", "rugo-1t"], + "rugo": [pypi, f"{HERE}/avro_one.py", "rugo"]} +ENV = {} +versions = {"rugo": sh(pypi, "-c", "import rugo;print(rugo.__version__)").strip(), + "rustc": sh("rustc", "--version").strip(), **crate_versions} +if a.rugo_src: + srcpy = venv("src", ["fastavro"]) + ENV = {**os.environ, "PYTHONPATH": os.path.abspath(a.rugo_src)} + ARMS["rugo-src-1t"] = [srcpy, f"{HERE}/avro_one.py", "rugo-1t"] + ARMS["rugo-src"] = [srcpy, f"{HERE}/avro_one.py", "rugo"] + versions["rugo-src"] = sh(srcpy, "-c", "import rugo;print(rugo.__version__, rugo.__file__)", env=ENV).strip() +ARMS.update({ + "arrow-avro": [BIN["default"], "arrow-avro"], + "arrow-avro-tuned": [BIN["zlib-rs"], "arrow-avro"], + "arrow-avro-tuned-view": [BIN["zlib-rs"], "arrow-avro"], + "apache-avro": [BIN["default"], "apache-avro"], +}) +RUST_TAIL = {"arrow-avro": ["1024", "0"], "arrow-avro-tuned": ["65536", "0"], + "arrow-avro-tuned-view": ["65536", "1"], "apache-avro": []} + +res = {"date": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"), + "machine": {"platform": platform.platform(), "cpu": platform.processor() or platform.machine(), + "cores": os.cpu_count(), "python": a.python}, + "rows": a.rows, "rounds": a.rounds, "reps": a.reps, "versions": versions, + "files_mb": {}, "reads": {}} + +for codec in ("deflate", "null"): + path = f"{W}/events-{codec}.avro" + print("generating", codec, flush=True) + sh(pypi, f"{HERE}/gen_avro.py", path, str(a.rows), codec) + res["files_mb"][codec] = round(os.path.getsize(path) / 1e6, 1) + for shape in ("all", "narrow"): + runs = {arm: [] for arm in ARMS} + names = list(ARMS) + for rnd in range(a.rounds): + order = names[rnd % len(names):] + names[:rnd % len(names)] + for arm in order: + print(f"read: {codec} {shape} round {rnd + 1}/{a.rounds} {arm}", flush=True) + cmd = ARMS[arm] + [shape, path, str(a.reps)] + RUST_TAIL.get(arm, []) + env = ENV if arm.startswith("rugo-src") else None + out = json.loads(sh(*cmd, env=env).strip().splitlines()[-1]) + if out["rows"] != a.rows: + raise SystemExit(f"{arm} read {out['rows']} rows, expected {a.rows}") + runs[arm].append(out) + for arm, outs in runs.items(): + res["reads"][f"{codec}/{shape}/{arm}"] = { + "best_ms": min(o["best_ms"] for o in outs), + "median_ms": round(statistics.median(o["median_ms"] for o in outs), 1), + "rows": a.rows} + +os.makedirs(f"{HERE}/results", exist_ok=True) +fn = f"{HERE}/results/{res['date'][:10]}-avro-rust.json" +json.dump(res, open(fn, "w"), indent=2) +print("wrote", fn) +for key, value in res["reads"].items(): + print(f" {key:40s} best {value['best_ms']:9.1f} median {value['median_ms']:9.1f} ms") diff --git a/benchmarks/results/2026-10-18-avro-rust.json b/benchmarks/results/2026-10-18-avro-rust.json new file mode 100644 index 0000000..ebccb87 --- /dev/null +++ b/benchmarks/results/2026-10-18-avro-rust.json @@ -0,0 +1,188 @@ +{ + "date": "2026-10-18T14:08:56+00:00", + "machine": { + "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", + "cpu": "arm", + "cores": 18, + "python": "3.14" + }, + "rows": 200000, + "rounds": 5, + "reps": 5, + "versions": { + "rugo": "0.5.4", + "rustc": "rustc 1.96.1 (31fca3adb 2026-06-26)", + "apache-avro": "0.22.0", + "arrow-avro": "60.0.0", + "flate2": "1.1.10", + "miniz_oxide": "0.9.1", + "zlib-rs": "0.6.5", + "rugo-src": "0.5.4 /Users/justin/Nextcloud/opteryx-core/rugo/__init__.py" + }, + "files_mb": { + "deflate": 21.6, + "null": 38.2 + }, + "reads": { + "deflate/all/rugo-1t": { + "best_ms": 84.1, + "median_ms": 85.1, + "rows": 200000 + }, + "deflate/all/rugo": { + "best_ms": 30.5, + "median_ms": 30.9, + "rows": 200000 + }, + "deflate/all/rugo-src-1t": { + "best_ms": 82.8, + "median_ms": 83.9, + "rows": 200000 + }, + "deflate/all/rugo-src": { + "best_ms": 30.1, + "median_ms": 30.3, + "rows": 200000 + }, + "deflate/all/arrow-avro": { + "best_ms": 116.4, + "median_ms": 117.6, + "rows": 200000 + }, + "deflate/all/arrow-avro-tuned": { + "best_ms": 101.4, + "median_ms": 102.3, + "rows": 200000 + }, + "deflate/all/arrow-avro-tuned-view": { + "best_ms": 113.1, + "median_ms": 114.7, + "rows": 200000 + }, + "deflate/all/apache-avro": { + "best_ms": 267.4, + "median_ms": 272.0, + "rows": 200000 + }, + "deflate/narrow/rugo-1t": { + "best_ms": 52.3, + "median_ms": 52.9, + "rows": 200000 + }, + "deflate/narrow/rugo": { + "best_ms": 18.2, + "median_ms": 18.3, + "rows": 200000 + }, + "deflate/narrow/rugo-src-1t": { + "best_ms": 51.1, + "median_ms": 51.5, + "rows": 200000 + }, + "deflate/narrow/rugo-src": { + "best_ms": 17.8, + "median_ms": 18.0, + "rows": 200000 + }, + "deflate/narrow/arrow-avro": { + "best_ms": 97.8, + "median_ms": 98.3, + "rows": 200000 + }, + "deflate/narrow/arrow-avro-tuned": { + "best_ms": 82.8, + "median_ms": 83.2, + "rows": 200000 + }, + "deflate/narrow/arrow-avro-tuned-view": { + "best_ms": 82.8, + "median_ms": 83.2, + "rows": 200000 + }, + "deflate/narrow/apache-avro": { + "best_ms": 398.0, + "median_ms": 402.0, + "rows": 200000 + }, + "null/all/rugo-1t": { + "best_ms": 46.3, + "median_ms": 47.9, + "rows": 200000 + }, + "null/all/rugo": { + "best_ms": 17.3, + "median_ms": 17.7, + "rows": 200000 + }, + "null/all/rugo-src-1t": { + "best_ms": 45.5, + "median_ms": 46.3, + "rows": 200000 + }, + "null/all/rugo-src": { + "best_ms": 17.1, + "median_ms": 17.4, + "rows": 200000 + }, + "null/all/arrow-avro": { + "best_ms": 40.2, + "median_ms": 40.7, + "rows": 200000 + }, + "null/all/arrow-avro-tuned": { + "best_ms": 40.2, + "median_ms": 40.6, + "rows": 200000 + }, + "null/all/arrow-avro-tuned-view": { + "best_ms": 51.0, + "median_ms": 51.6, + "rows": 200000 + }, + "null/all/apache-avro": { + "best_ms": 200.4, + "median_ms": 204.3, + "rows": 200000 + }, + "null/narrow/rugo-1t": { + "best_ms": 14.2, + "median_ms": 14.5, + "rows": 200000 + }, + "null/narrow/rugo": { + "best_ms": 5.0, + "median_ms": 5.1, + "rows": 200000 + }, + "null/narrow/rugo-src-1t": { + "best_ms": 13.9, + "median_ms": 14.2, + "rows": 200000 + }, + "null/narrow/rugo-src": { + "best_ms": 4.8, + "median_ms": 4.9, + "rows": 200000 + }, + "null/narrow/arrow-avro": { + "best_ms": 20.6, + "median_ms": 21.1, + "rows": 200000 + }, + "null/narrow/arrow-avro-tuned": { + "best_ms": 20.7, + "median_ms": 21.2, + "rows": 200000 + }, + "null/narrow/arrow-avro-tuned-view": { + "best_ms": 20.8, + "median_ms": 21.0, + "rows": 200000 + }, + "null/narrow/apache-avro": { + "best_ms": 328.4, + "median_ms": 335.5, + "rows": 200000 + } + } +} \ No newline at end of file diff --git a/docs/blog/index.html b/docs/blog/index.html index bccbf3b..cca54a3 100644 --- a/docs/blog/index.html +++ b/docs/blog/index.html @@ -31,6 +31,17 @@

Blog

PyArrow, pandas or NumPy. Measurements, not estimates.

+ +

Performant Python pipelines without the Rust

+

18 October 2026

+

+ rugo's Avro reader, called from Python, against Apache + Arrow's Rust reader. On one thread Python won three of + four races, by 1.2–1.6×. On its defaults rugo was + 2.3–4.5× faster than any Rust result. +

+ +

When rugo makes your Lambda cheaper, and when it diff --git a/docs/blog/performant-python-pipelines-without-the-rust.html b/docs/blog/performant-python-pipelines-without-the-rust.html new file mode 100644 index 0000000..b13643a --- /dev/null +++ b/docs/blog/performant-python-pipelines-without-the-rust.html @@ -0,0 +1,445 @@ + + + + + + Performant Python pipelines without the Rust + + + + + + + +
+
+
+

Performant Python pipelines without the Rust

+

+ Justin Joyce · 18 October 2026 + · 6 min read +

+
+ avro + rust + performance + benchmarks +
+

+ Correct at time of publishing. For up-to-date + performance and feature information, see + rugo.dev. +

+
+ +

+ Called from Python, rugo reads Avro faster than + arrow-avro, Apache Arrow's own Rust reader. + On one thread it won three of four races, + by 1.2× to 1.6×. Left on its defaults, it uses every core + and was 2.3× to 4.5× faster than any Rust + result. +

+ +

+ There's a familiar arc in data engineering. The Python + pipeline gets slow, someone benchmarks it, and the + conclusion is always the same: rewrite the hot path in Rust. + Weeks later you have a second language, a build toolchain, + FFI glue, and a team that can't all read the code. +

+ +

+ We think that trade is usually unnecessary. To test it, we + raced rugo's Avro reader, called from Python, against the + best Avro readers Rust has, called from Rust. +

+ +

The contenders

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
ReaderCalled fromReturns
rugo 0.5.4Pythoncolumnar Morsels
+ arrow-avro + 60.0.0 + RustArrow RecordBatches
apache-avro 0.22.0Rustone Value per record
+
+ +

+ arrow-avro is the one to beat. It decodes straight into + columns, as rugo does, and in our runs it was 2.7× to 16× + faster than the older apache-avro crate. We ran it twice: + once on its defaults, and once tuned as far as we could push + it, with 65,536-row batches and its deflate switched to + zlib-rs, a faster Rust inflater than its default. +

+ +

The test

+ +

+ One file: 200,000 rows of a 20-field event record, the mix + you'd see in a Kafka topic or a log stream. Longs, doubles, + strings, enums, nullable unions, timestamps, dates, a + decimal, a fixed(16), a nested record, an array + and a map. fastavro wrote it, so no reader is timed on its + own output. +

+ +

+ We wrote it twice, once with deflate (21.6 MB) and once + uncompressed (38.2 MB), and read it two ways: +

+ +
    +
  • all fields: every one of the 20.
    • +
  • + 3 fields: id, + amount and country. rugo is + given columns; arrow-avro a projection; + apache-avro a reader schema of just those fields. +
    • +
+ +

+ Every reader decoded the same in-memory bytes, so disk wasn't + part of the timing. Each run was a fresh process on one + thread. The readers took turns across five rounds, and the + figures are medians. The machine is an 18-core Apple Silicon + Mac. +

+ +

One thread each

+ +

Median read time, shorter is faster:

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Racerugo (Python)arrow-avroapache-avrorugo vs arrow-avro
deflate, all fields85.1 ms102.3 ms272.0 ms1.20× faster
deflate, 3 fields52.9 ms83.2 ms402.0 ms1.57× faster
uncompressed, all fields47.9 ms40.6 ms204.3 ms1.18× slower
uncompressed, 3 fields14.5 ms21.0 ms335.5 ms1.45× faster
+
+ +

+ Python won three of the four. The arrow-avro figures are its + tuned build; on its defaults it was 15 ms slower on the + deflate races and level on the uncompressed ones. + apache-avro was never in it: it came last in every race, up + to 23× slower than rugo. +

+ +

Every core

+ +

+ One thread each keeps the race fair, but it isn't how anyone + runs rugo. On its defaults, rugo spreads a file's blocks + across every core. Neither Rust reader can do that for a + single file, so this compares rugo, untouched, with + arrow-avro at its best: +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Racerugo, defaultsarrow-avro, tunedrugo vs arrow-avro
deflate, all fields30.9 ms102.3 ms3.3× faster
deflate, 3 fields18.3 ms83.2 ms4.5× faster
uncompressed, all fields17.7 ms40.6 ms2.3× faster
uncompressed, 3 fields5.1 ms21.0 ms4.1× faster
+
+ +

That's what you get without changing a setting.

+ +

Where we lost

+ +

+ One race went to Rust. Reading every field of the + uncompressed file on one thread, arrow-avro took 40.6 ms and + rugo took 47.9 ms, so arrow-avro was 18% faster. +

+ +

+ We're showing it because it's the purest test of the decoder + itself: no inflate, no skipping, just bytes into columns. + It's also the one we most want to win. +

+ +

+ Our best guess, not yet tested: rugo turns the nested record + and the map into JSON text, which is extra work. arrow-avro + builds native struct and map arrays instead. Reading the file + without those two fields would tell us whether the decoder + itself is behind. +

+ +

+ It doesn't change the headline. Kafka Connect, Spark and + Iceberg usually write Avro compressed, and on compressed + files Python wins. +

+ +

Why Python can be this fast

+ +

+ Python isn't doing the work. rugo is a native C++ engine with + a Python front door: you call it from Python, and the bytes + never pass through the interpreter. +

+ +
    +
  • + Compiled schema. rugo turns the Avro + schema into a flat program of decode steps once, then + runs that program over every block. Nothing is looked up + per row. +
    • +
  • + Straight into columns. Values land in + typed columnar vectors, with no Python object per value. + That's what you'd write in Rust; you just don't have to. +
    • +
  • + Fast inflate. Deflate blocks go through + libdeflate, a dedicated, heavily tuned inflate library. + On compressed files that's most of the time, and most of + the lead. +
    • +
  • + Cheap skipping. Fields you don't ask for + are stepped over, not decoded. On the uncompressed file, + reading 3 of 20 fields takes about a third of the time + of reading them all. +
    • +
  • + Every core. A file's blocks are + independent, so rugo spreads them across threads and + still hands back batches in file order. +
    • +
+ +

+ The point isn't that Rust is slow. arrow-avro is excellent + work. The point is that "fast" is a property of the engine, + not of the language you call it from. +

+ +

What this leaves out

+ +
    +
  • + One schema. A wide, flat record, or one + that's mostly strings, could move the margins either way. +
    • +
  • + One machine. Every figure here is from + Apple Silicon. We haven't run x86 yet. +
    • +
  • + Different outputs. rugo returns nested + records and maps as JSON text; arrow-avro returns native + struct and map arrays. Both end in columns, but they + aren't the same columns. +
    • +
  • + Other languages. We didn't test Go, + Java or C++ readers. The ones we know of decode a row at + a time, as apache-avro does. +
    • +
  • + Reading the file. Every reader got the + bytes already in memory. Reading from disk or object + storage adds the same time to each. +
    • +
+ +

Try it

+ +

+ pip install rugo is the whole install. There's + no toolchain to set up, and no PyArrow or NumPy: +

+ + 
from rugo.avro import read_avro
+
+with read_avro(
+    "events.avro", columns=["id", "amount", "country"]
+) as reader:
+    for morsel in reader:
+        print(morsel.num_rows, morsel.column("amount").to_pylist()[:3])
+ +

Reproducing this

+ +

+ Measured 18 October 2026 on an 18-core Apple Silicon Mac, + against rugo 0.5.4 from PyPI on CPython 3.14, and arrow-avro + 60.0.0 and apache-avro 0.22.0 built with Rust 1.96.1 + (release profile, fat LTO). The tuned arrow-avro build + switches flate2 to zlib-rs + 0.6.5; the default build uses miniz_oxide. + rugo runs with max_threads=1 for the + one-thread races. +

+ +

+ Each of the five rounds runs every reader in a fresh + process, five reads each, with the order rotated each round + so no reader always goes first. The figures are the median + of the per-process medians. +

+ +

+ The harness, the data generator and the Rust readers are in + benchmarks: measure_avro_rust.py builds both Rust + readers, writes the file and runs the race on your machine. + We'd like to hear what you get, especially on x86. +

+
+
+ +
+
+ rugo is part of the + Opteryx project — Apache + 2.0 + + GitHub + · + PyPI + +
+
+ + + + diff --git a/medium/performant-python-pipelines-without-the-rust/table-1.png b/medium/performant-python-pipelines-without-the-rust/table-1.png new file mode 100644 index 0000000..84f8135 Binary files /dev/null and b/medium/performant-python-pipelines-without-the-rust/table-1.png differ diff --git a/medium/performant-python-pipelines-without-the-rust/table-2.png b/medium/performant-python-pipelines-without-the-rust/table-2.png new file mode 100644 index 0000000..196e1d9 Binary files /dev/null and b/medium/performant-python-pipelines-without-the-rust/table-2.png differ diff --git a/medium/performant-python-pipelines-without-the-rust/table-3.png b/medium/performant-python-pipelines-without-the-rust/table-3.png new file mode 100644 index 0000000..750f200 Binary files /dev/null and b/medium/performant-python-pipelines-without-the-rust/table-3.png differ diff --git a/tools/medium_table_images.py b/tools/medium_table_images.py new file mode 100644 index 0000000..7d6b1c9 --- /dev/null +++ b/tools/medium_table_images.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Render every table in a rugo.dev blog post to a PNG, for Medium. + + uv run --with pillow python tools/medium_table_images.py docs/blog/.html + +Medium has no tables: a story imported from rugo.dev keeps its headings, text and +code, but each table has to go back in as an image. This renders each +`
` in the post on its own page with the blog's own +stylesheet (headless Microsoft Edge, 2x scale), trims the empty background, and +writes /table-1.png, table-2.png, ... in page order, printing which +section each one belongs under. +""" +import os, re, subprocess, sys, tempfile + +from PIL import Image + +EDGE = "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge" +DOCS = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "docs")) +PAD = 32 # px of background kept round the table, at 2x scale + +post, out = sys.argv[1], sys.argv[2] +html = open(post, encoding="utf-8").read() +os.makedirs(out, exist_ok=True) + +# Each table with the last

above it, so the caller knows where it goes. +tables = [] +for m in re.finditer(r'
.*?\s*
', html, re.S): + headings = re.findall(r"

(.*?)

", html[: m.start()], re.S) + tables.append((headings[-1].strip() if headings else "(top)", m.group(0))) +if not tables: + sys.exit(f"no tables in {post}") + +with tempfile.TemporaryDirectory() as tmp: + for n, (section, table) in enumerate(tables, 1): + page = f"{tmp}/table-{n}.html" + with open(page, "w", encoding="utf-8") as f: + f.write(f""" + + +{table}""") + raw = f"{tmp}/raw-{n}.png" + subprocess.run([EDGE, "--headless=new", "--disable-gpu", "--hide-scrollbars", + "--force-device-scale-factor=2", "--window-size=760,1600", + f"--screenshot={raw}", f"file://{page}"], + check=True, capture_output=True) + img = Image.open(raw).convert("RGB") + bg = img.getpixel((0, 0)) + diff = [(x, y) for y in range(0, img.height, 2) for x in range(0, img.width, 4) + if img.getpixel((x, y)) != bg] + if not diff: + sys.exit(f"table {n} rendered empty") + x0 = max(min(x for x, _ in diff) - PAD, 0) + y0 = max(min(y for _, y in diff) - PAD, 0) + x1 = min(max(x for x, _ in diff) + PAD, img.width) + y1 = min(max(y for _, y in diff) + PAD, img.height) + dest = f"{out}/table-{n}.png" + img.crop((x0, y0, x1, y1)).save(dest) + print(f"{dest} -> under '{section}'")