diff --git a/Cargo.lock b/Cargo.lock index df0e730..75690d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,20 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -20,6 +34,180 @@ dependencies = [ "libc", ] +[[package]] +name = "arrow" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" + +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -31,12 +219,27 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bitflags" version = "2.13.1" @@ -82,6 +285,26 @@ dependencies = [ "windows-link", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.11.0" @@ -131,6 +354,12 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "ctor" version = "1.0.13" @@ -172,6 +401,16 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -290,6 +529,41 @@ dependencies = [ "windows-result", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -299,13 +573,19 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown", + "hashbrown 0.15.5", ] [[package]] @@ -484,6 +764,63 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.189" @@ -500,6 +837,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -633,6 +976,34 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -640,6 +1011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -739,6 +1111,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -748,6 +1126,18 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.18" @@ -809,6 +1199,12 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "scoped-tls" version = "1.0.1" @@ -951,6 +1347,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -1058,6 +1463,27 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -1191,6 +1617,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.4" @@ -1220,6 +1652,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zerofrom" version = "0.1.8" @@ -1277,7 +1729,7 @@ dependencies = [ [[package]] name = "zu" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "zu-common", "zu-encoding", @@ -1290,10 +1742,20 @@ dependencies = [ "zu-zu1", ] +[[package]] +name = "zu-arrow" +version = "0.0.1" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" +dependencies = [ + "arrow", + "zu-common", + "zu-query", +] + [[package]] name = "zu-common" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "thiserror", ] @@ -1301,7 +1763,7 @@ dependencies = [ [[package]] name = "zu-encoding" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "ruzstd", "zu-common", @@ -1310,7 +1772,7 @@ dependencies = [ [[package]] name = "zu-exec" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "zu-common", "zu-query", @@ -1320,7 +1782,7 @@ dependencies = [ [[package]] name = "zu-query" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "crossbeam-deque", "zu-common", @@ -1331,7 +1793,7 @@ dependencies = [ [[package]] name = "zu-s3" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "crc32c", "object_store", @@ -1342,7 +1804,7 @@ dependencies = [ [[package]] name = "zu-sqlite" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "rusqlite", "zu-common", @@ -1352,7 +1814,7 @@ dependencies = [ [[package]] name = "zu-storage" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "zu-common", "zu-encoding", @@ -1361,7 +1823,7 @@ dependencies = [ [[package]] name = "zu-vector" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "zu-common", ] @@ -1369,7 +1831,7 @@ dependencies = [ [[package]] name = "zu-zu1" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=6753ded13a215bf5e8fe70ce41e69b35321e3e73#6753ded13a215bf5e8fe70ce41e69b35321e3e73" +source = "git+https://github.com/tamnd/zu?rev=0698a4eccd31670f0a875b6d097d7753440a51b3#0698a4eccd31670f0a875b6d097d7753440a51b3" dependencies = [ "crc32c", "loom", @@ -1385,5 +1847,6 @@ dependencies = [ "napi-build", "napi-derive", "zu", + "zu-arrow", "zu-common", ] diff --git a/Cargo.toml b/Cargo.toml index 7699ce1..b1a3191 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,14 @@ crate-type = ["cdylib"] # with (ADR 0002), so a revision is the honest way to say which one. # A local checkout is used instead with a `paths` override in # `.cargo/config.toml`, which is untracked on purpose. -zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "6753ded13a215bf5e8fe70ce41e69b35321e3e73" } -zu-common = { git = "https://github.com/tamnd/zu", rev = "6753ded13a215bf5e8fe70ce41e69b35321e3e73" } +zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } +zu-common = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3" } +# The one translation from a result into Arrow, which lives in the engine +# tree so that every client agrees about what a column becomes. `ipc` is +# the only feature this client turns on: the C Data Interface hands over +# a pointer and nothing in a JavaScript runtime can read one, so the way +# out here is the bytes of an IPC stream. +zu-arrow = { git = "https://github.com/tamnd/zu", rev = "0698a4eccd31670f0a875b6d097d7753440a51b3", features = ["ipc"] } # N-API by way of napi-rs (ADR 0002). `napi9` is the version of N-API # this addon declares it needs, which is what makes one binary work # across Node 24, Node 26, Electron and Bun without a rebuild: the diff --git a/README.md b/README.md index 0f54de8..2d08d58 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ The switches come across, including `bigIntMode` and `temporal`, because a pool ## What works today -`connect`, `query`, `exec`, `stream`, `close`, `dispose` and `await using`. `duplicate`, for a second connection made from the first. Named parameters both ways, including lists, records and nesting. Every scalar the engine has, plus nodes, edges and paths with their tables named rather than numbered, and `ZuDate`, `ZuTime`, `ZuTimestamp` and `ZuDuration`, with `{ temporal: true }` and `toTemporal()` for the runtimes that have `Temporal`. Read-only connections, databases in memory, memory and thread limits. `bigIntMode`, per statement or per connection. An `AbortSignal` on any statement, and `rowsRead` and `progress` for watching the one running now. The full error surface above, and `isZuError` to recognize it. Streaming, as an async iterable, as batches and as a Web Stream. Transactions, with `inTransaction` on the connection. An appender, for loading rows a batch at a time, and `load` for building a whole database out of columns and an edge list. Registered frames, so an Arrow table or an object of typed arrays is something a statement can match on without the rows being copied. `columnar`, for a result read down its columns as the buffers themselves rather than across its rows as objects. Prepared statements, compiled at the line that asked and run as often as wanted, and `explain` and `profile`, as a tree a program walks and as the listing a person reads. Both module formats, typed separately. +`connect`, `query`, `exec`, `stream`, `close`, `dispose` and `await using`. `duplicate`, for a second connection made from the first. Named parameters both ways, including lists, records and nesting. Every scalar the engine has, plus nodes, edges and paths with their tables named rather than numbered, and `ZuDate`, `ZuTime`, `ZuTimestamp` and `ZuDuration`, with `{ temporal: true }` and `toTemporal()` for the runtimes that have `Temporal`. Read-only connections, databases in memory, memory and thread limits. `bigIntMode`, per statement or per connection. An `AbortSignal` on any statement, and `rowsRead` and `progress` for watching the one running now. The full error surface above, and `isZuError` to recognize it. Streaming, as an async iterable, as batches and as a Web Stream. Transactions, with `inTransaction` on the connection. An appender, for loading rows a batch at a time, and `load` for building a whole database out of columns and an edge list. Registered frames, so an Arrow table or an object of typed arrays is something a statement can match on without the rows being copied. `columnar`, for a result read down its columns as the buffers themselves rather than across its rows as objects, and `arrow`, for the same result as an Arrow IPC stream that any Arrow reader takes. Prepared statements, compiled at the line that asked and run as often as wanted, and `explain` and `profile`, as a tree a program walks and as the listing a person reads. Both module formats, typed separately. Build it with `npm run build`, and run the suite with `npm test`. Nothing is published yet, so `npm i zudb` is not a thing you can type at anybody's terminal, but everything it will do is built and installed on every run of the release workflow. @@ -251,15 +251,15 @@ read.columns[0].values; // a BigInt64Array of every age, and not one object The buffers are the engine's own, moved rather than read: the pointer V8 is given is the pointer the engine filled, and the allocation is freed when the typed array is collected. So a column of a million integers crosses the boundary as a pointer and a length. On this machine, with `npm run bench:columnar` over a million rows: ``` -one integer column, columnar 38.4 ms 38 ns/row -one integer column, rows 243.1 ms 243 ns/row -a string column, columnar 50.3 ms 50 ns/row -a string column, rows 262.0 ms 262 ns/row -three columns, columnar 75.8 ms 76 ns/row -three columns, rows 632.2 ms 632 ns/row +one integer column, columnar 2.6 ms 3 ns/row +one integer column, rows 240.7 ms 241 ns/row +a string column, columnar 12.4 ms 12 ns/row +a string column, rows 253.1 ms 253 ns/row +three columns, columnar 16.0 ms 16 ns/row +three columns, rows 608.7 ms 609 ns/row ``` -Walking what came back costs the same either way, at about 14 ns a row for a sum over the buffer and the same over the rows, which is worth saying because it is where the win is not. V8 reads a property of a small object about as fast as an element of a typed array. What it cannot do is make a million of those objects for nothing, and that is the whole of the six to eight times above. +Walking what came back costs the same either way, at about 14 ns a row for a sum over the buffer and the same over the rows, which is worth saying because it is where the win is not. V8 reads a property of a small object about as fast as an element of a typed array. What it cannot do is make a million of those objects for nothing, and it is not the only thing not happening on the columnar line: the engine's sink fills those buffers as the statement runs, so a columnar read is a scan and a move, where a row read is a scan, a row per row and then an object per row on top of it. That is the whole of the twenty to ninety times above, and it is why the two lines of a pair are further apart the more columns there are. Every column says what it is, and reading one is a switch on `type` rather than a series of tests for what is there. `values` carries everything of a fixed width: a `BigInt64Array` of integers, nanoseconds or months, a `Float64Array` of floats, an `Int32Array` of days, and for booleans a `Uint8Array` of one bit a row, least significant bit first. A string column has `data`, the bytes of every string end to end, and `offsets`, one more than there are rows, so row `i` is `data.subarray(offsets[i], offsets[i + 1])`. `validity` is one bit a row again, set meaning the row has a value, and it is null when nothing in the column is, so the common case costs a reader nothing to skip. `unit` says whether a cell counts days, nanoseconds or months, and `zone` is the minutes east of UTC a column of zoned times was written with. @@ -294,6 +294,61 @@ A statement that matched nothing still comes back with its columns, each one the `bigIntMode` says nothing here. A columnar read has one physical layout per type and an INT64 column is 64 bit cells however a caller would rather read one, which is the difference between a buffer and a value. The mode still decides what is inside `items`, where this client is making objects anyway. +## Reading a result as Arrow + +`columnar` hands over the buffers and leaves the reader to say what each one means, which is eleven lines of Arrow before there is a table. `arrow` runs the same statement and writes those same buffers into an Arrow IPC stream, schema first, so what comes back is a result every Arrow implementation already reads: + +```ts +// with apache-arrow installed, and nothing in zudb importing it +import { tableFromIPC } from "apache-arrow"; + +const read = await conn.arrow(`MATCH (p:person) RETURN p.name AS name, p.age AS age`); +const table = tableFromIPC(read.ipc); +table.numRows; // 1000000 +table.getChild("age").get(0); // 42n +``` + +`read.ipc` is a `Uint8Array`, and it is the addon's own allocation handed over rather than copied. Being bytes rather than an object is what makes it go places an object does not: it posts to a worker as a transferable instead of being cloned, it is a `fetch` body as it stands, it is what DuckDB-Wasm and every Arrow reader in the browser take, and it can be written to a file that `pyarrow` opens on the other side of the world. The framing is what that costs, from `npm run bench:arrow` over a million rows: + +``` +one integer column, arrow 8.5 ms 8 ns/row +one integer column, columnar 3.8 ms 4 ns/row +one integer column, rows 276.0 ms 276 ns/row +three columns, arrow 24.1 ms 24 ns/row +three columns, columnar 18.2 ms 18 ns/row +three columns, rows 703.8 ms 704 ns/row +``` + +So the stream costs about a third more than handing the raw buffers over, and about a thirtieth of what building row objects costs. Reading it back is 0.3 ms for those three columns, because `tableFromIPC` reads the headers and points at the bytes rather than walking them. Take `columnar` when the next thing that happens to the numbers is a loop you are writing, and take `arrow` when the result is going somewhere that already speaks Arrow. + +Every zu type has one Arrow type, and it is the same one in every zu client: + +| zu | Arrow | +|---|---| +| INT64 | `Int64` | +| FLOAT64 | `Float64` | +| BOOL | `Bool` | +| STRING | `Utf8`, or `LargeUtf8` past what a 32 bit offset addresses | +| DATE | `Date32` | +| LOCAL TIME | `Time64` | +| LOCAL DATETIME | `Timestamp` | +| ZONED DATETIME | `Timestamp` with the offset on the field | +| DURATION, day-time | `Duration` | +| DURATION, year-month | `Interval` | +| a node | a struct of `table` and `offset` | +| an edge | a struct of `table`, `src`, `dst` and `ord` | +| a path | a struct of `nodes` and `rels`, each a list of the above | +| a list | an Arrow list | +| a record | an Arrow struct | + +A path is two lists rather than one because Arrow has no type for a list whose elements alternate between two shapes, and a walk is the nodes it visited and the edges it crossed either way, one more node than edge. A node names its table rather than numbering it, which is the same name `query` puts on a `ZuNode`. + +Two things have no Arrow type at all and are refused by name rather than translated into something near enough. A `ZONED TIME` is a time with an offset, and Arrow has a time and a timestamp and nothing in between, so dropping the offset would move the value: read that column with `columnar`, where the cells and the offset arrive side by side. A handle to a graph or to a binding table is a reference to something that is not in the result, so there is nothing to put in a column. A column that mixes two types is refused the same way it is for `columnar`, naming the column and the row that did it. + +`batchRows` says how many rows go in one record batch, and it is 65,536 by default. The arrays are built whole and a batch is a slice of them, so the size costs nothing to change and is worth naming only when the reader on the other side wants a particular one. A statement that matched nothing is a schema and one empty batch rather than no bytes at all, so the columns and their types are known either way, and a table built from an empty answer concatenates with one built from a full answer. + +The translation itself is not in this package. It lives in the engine, beside the query planner, and the Python client exports through the same code, so a year-month duration is a month interval in both and a node names its table in both. That is deliberate: two copies of it would be two sets of rules about what a duration is, and the second one would drift. + ## Preparing a statement `conn.prepare` compiles a statement now and hands back something that runs it later, as often as it is asked to, with different values bound each time: @@ -305,7 +360,7 @@ const ada = await find.query<{ id: bigint }>({ name: "ada" }); const zoe = await find.query<{ id: bigint }>({ name: "zoe" }); ``` -It answers the same three ways a connection does. `query` gives rows, `exec` gives nothing and is for a statement written to change something, and `columnar` gives the buffers. Each takes the bindings and the same options a statement takes, so a signal and a `bigIntMode` go on the run rather than on the prepare, since which run a caller wants to stop is a property of that run. +It answers the same four ways a connection does. `query` gives rows, `exec` gives nothing and is for a statement written to change something, `columnar` gives the buffers and `arrow` gives the IPC stream. Each takes the bindings and the same options a statement takes, so a signal and a `bigIntMode` go on the run rather than on the prepare, since which run a caller wants to stop is a property of that run. What this is not is a speedup, and it is worth saying so here rather than letting a reader assume the thing every other client's documentation says. A driver prepares to save a round trip to a server, and there is no server and no round trip here. The engine already caches the plan for a statement by its text, so the second `conn.query` of the same string is not compiled a second time either. On this machine, with `npm run bench:prepared` over 100 rows and 5000 runs: @@ -446,7 +501,7 @@ typedoc rather than api-documenter, which would have been the obvious pick since Anything outside that table has no binary and no source build to fall back on, so the install resolves nothing and the first `require` says so. The browser and the platforms nobody builds for are what the WASM target answers, later. -`npm run bench` measures what this package adds to the engine, which is a row object and one JavaScript value per column: the same scan with the rows dropped is the floor, and the difference between the two is what the boundary costs. Run it against a release build, since a debug build of the engine moves the floor by an order of magnitude and not the rest of it. `npm run bench:append` does the same for the appender, `npm run bench:load` for building a database out of columns, `npm run bench:register` for registered frames, where what is being watched is that the registration does not scale with the rows, `npm run bench:columnar` for a result read down its columns against the same result read across its rows, and `npm run bench:prepared` for a prepared statement against the same text run again and against a text that is new every time, which is the one of these whose interesting number is that the first two are equal. +`npm run bench` measures what this package adds to the engine, which is a row object and one JavaScript value per column: the same scan with the rows dropped is the floor, and the difference between the two is what the boundary costs. Run it against a release build, since a debug build of the engine moves the floor by an order of magnitude and not the rest of it. `npm run bench:append` does the same for the appender, `npm run bench:load` for building a database out of columns, `npm run bench:register` for registered frames, where what is being watched is that the registration does not scale with the rows, `npm run bench:columnar` for a result read down its columns against the same result read across its rows, `npm run bench:arrow` for the same result as an Arrow stream against both of those, where the number being watched is what the framing costs over handing the buffers straight over, and `npm run bench:prepared` for a prepared statement against the same text run again and against a text that is new every time, which is the one of these whose interesting number is that the first two are equal. ## Still to come @@ -462,7 +517,7 @@ Bun and Deno in CI, and the WASM build for the browser. | Browser and edge | `zudb/wasm` | read-mostly, over OPFS or HTTP range requests | | Electron | the same binary | N-API is ABI-stable across Electron versions, so no per-Electron rebuild | -`apache-arrow` is a dev dependency and not a dependency, and it is one so that the tests can build the tables the register path reads. Nothing in the package imports it, so a caller who never registers a frame never installs it. +`apache-arrow` is a dev dependency and not a dependency, and it is one so that the tests can build the tables the register path reads and read back the streams `arrow` writes. Nothing in the package imports it: a frame is recognized by its shape and a stream is bytes, so a caller who wants neither never installs it, and one who does is free to install whichever version of Arrow they were going to use anyway. One binary serves all three, because N-API is the ABI all three implement, and the whole suite runs on each of them in CI rather than the other two being assumed from Node passing. `npm run test:bun` and `npm run test:deno` run it locally. What the three do not agree on is what a native error carries: V8 writes a `stack` when the error is made and JavaScriptCore writes none at all through N-API, so this client writes the header line itself when it finds none, non-enumerably, and `err.stack` starts with the condition's name on all of them. diff --git a/bench/arrow.mjs b/bench/arrow.mjs new file mode 100644 index 0000000..741a3e5 --- /dev/null +++ b/bench/arrow.mjs @@ -0,0 +1,96 @@ +// What a result costs as Arrow, against the two other ways out. +// +// Three calls over the same statement. `query` builds an object a row +// and a value a cell. `columnar` moves one buffer a column and leaves +// the reader to put a type around them. `arrow` writes those same +// buffers into an IPC stream, which costs a copy of the values and a +// header a batch, and buys a result every Arrow implementation reads. +// +// So the pair worth reading is `arrow` against `columnar`: the +// difference between them is what the framing costs, and it is the +// number that says whether a caller should take the bytes or take the +// buffers. The line after each is the reader's side, since bytes nobody +// decodes are not a result: `tableFromIPC` against the eleven lines the +// README prints for wrapping the buffers. +// +// Run it against a release build, for the reason bench/query.mjs gives. +// +// npm run build && npm run bench:arrow + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { tableFromIPC } from 'apache-arrow' +import { connect } from 'zudb' + +const ROWS = Number(process.env.ZU_BENCH_ROWS ?? 1_000_000) +const REPEATS = Number(process.env.ZU_BENCH_REPEATS ?? 5) + +const dir = await mkdtemp(join(tmpdir(), 'zu-bench-arrow-')) +const conn = await connect(join(dir, 'bench.zu1')) + +await conn.exec("INSERT (p:person {uid: 1, score: 1.5, name: 'n1'})") +{ + const rows = await conn.appender('person') + for (let ix = 2; ix <= ROWS; ix++) rows.appendRow([BigInt(ix), ix / 3, `n${ix}`]) + await rows.close() +} + +// The fastest of `REPEATS` runs, in milliseconds, after one warmup, for +// the reason bench/query.mjs gives: everything that makes a run slower +// than the work itself is something that happened to it rather than +// something about it. +async function time(run) { + await run() + let best = Infinity + for (let round = 0; round < REPEATS; round++) { + const started = performance.now() + await run() + best = Math.min(best, performance.now() - started) + } + return best +} + +function report(name, ms) { + const each = (ms * 1e6) / ROWS + console.log( + `${name.padEnd(30)} ${ms.toFixed(1).padStart(8)} ms ${Math.round(each).toString().padStart(6)} ns/row`, + ) +} + +const ONE = 'MATCH (p:person) RETURN p.uid AS uid' +const THREE = 'MATCH (p:person) RETURN p.uid AS uid, p.score AS score, p.name AS name' + +const cases = [ + { name: 'one integer column, arrow', run: () => conn.arrow(ONE) }, + { name: 'one integer column, columnar', run: () => conn.columnar(ONE) }, + { name: 'one integer column, rows', run: () => conn.query(ONE) }, + { name: 'three columns, arrow', run: () => conn.arrow(THREE) }, + { name: 'three columns, columnar', run: () => conn.columnar(THREE) }, + { name: 'three columns, rows', run: () => conn.query(THREE) }, +] + +console.log(`reading ${ROWS} rows, fastest of ${REPEATS}`) +for (const { name, run } of cases) report(name, await time(run)) + +// What a caller does next, which is the half the calls above do not +// include. The bytes are read once here and decoded every round, so what +// is timed is the decode and not the statement. +const bytes = (await conn.arrow(THREE)).ipc + +console.log('') +console.log('turning what came back into a table') +report('tableFromIPC over the bytes', await time(async () => tableFromIPC(bytes))) + +// A batch is a slice of arrays that are already built, so the size is +// about what the reader holds at once rather than about the write. This +// says by how much, which is the answer to whether it is worth tuning. +console.log('') +console.log('the batch size the stream is cut into') +for (const batchRows of [4_096, 65_536, 1_000_000]) { + report(`batchRows ${batchRows}`, await time(() => conn.arrow(THREE, null, { batchRows }))) +} + +await conn.close() +await rm(dir, { recursive: true, force: true }) diff --git a/binding.d.cts b/binding.d.cts index fd91a7d..a79cbbc 100644 --- a/binding.d.cts +++ b/binding.d.cts @@ -377,6 +377,46 @@ export interface ZuColumnar { readonly notices: ZuNotice[] } +/** + * A whole result as Arrow, in the bytes Arrow ships between processes. + * + * The same buffers a columnar read hands over, with the schema written + * beside them, so `tableFromIPC(read.ipc)` is the whole of the reading + * code and every Arrow implementation is a reader. A result with no rows + * is a schema and one empty batch rather than nothing at all, so the + * columns are known either way. + */ +export interface ZuArrow { + /** + * The stream, as one buffer: a schema message and then a message a + * batch. It is the addon's own allocation handed over rather than + * copied, and it detaches when posted to a worker, which is what makes + * a result cross a thread without being cloned. + */ + readonly ipc: Uint8Array + /** How many rows are in it, which the batches also add up to. */ + readonly rows: number + readonly gqlstatus: string + readonly notices: ZuNotice[] +} + +/** + * What a statement read as Arrow takes beside its parameters. + */ +export interface ZuArrowOptions extends ZuStatementOptions { + /** + * How many rows one record batch holds. Arrow's own 65,536 by + * default, which is what a reader expects and what keeps a batch + * inside a cache. + * + * The arrays are built whole either way and a batch is a slice of + * them, so this costs nothing to change and buys nothing to tune. It + * is worth naming when the reader on the other side has a size of its + * own, or when the batches are going somewhere one at a time. + */ + readonly batchRows?: number +} + /** * The rows a statement gave back. * @@ -998,6 +1038,29 @@ export declare class Connection { * JavaScript values on the way. */ columnar(statement: string, params?: Record | null, options?: ZuStatementOptions | null): Promise + /** + * Runs one statement and gives back the bytes of an Arrow IPC + * stream. + * + * ```js + * import { tableFromIPC } from 'apache-arrow' + * const read = await conn.arrow('MATCH (a:account) RETURN a.name AS name, a.balance AS balance') + * const table = tableFromIPC(read.ipc) + * ``` + * + * The same buffers [`Connection::columnar`] hands over, with the + * schema written beside them in the format every Arrow + * implementation already reads. That is the difference worth + * knowing: `columnar` is the fastest way out and leaves the caller + * to say what each buffer means, and this is the one where the + * result arrives as a table, a dataframe or a DuckDB relation with + * no code in between. + * + * The translation lives in the engine and is the same one the + * Python client exports through, so a node column names its table + * in both and a year-month duration is a month interval in both. + */ + arrow(statement: string, params?: Record | null, options?: ZuArrowOptions | null): Promise /** * Compiles a statement, pins it, and hands back something that * runs it. @@ -1166,6 +1229,8 @@ export declare class Prepared { exec(params?: Record | null, options?: ZuStatementOptions | null): Promise /** Runs it and gives back its columns rather than its rows. */ columnar(params?: Record | null, options?: ZuStatementOptions | null): Promise + /** Runs it and gives back the bytes of an Arrow IPC stream. */ + arrow(params?: Record | null, options?: ZuArrowOptions | null): Promise /** * Gives the id back to the session. * diff --git a/etc/zudb.api.md b/etc/zudb.api.md index 1f1ae18..30ff55c 100644 --- a/etc/zudb.api.md +++ b/etc/zudb.api.md @@ -27,6 +27,7 @@ export function connect(path?: string | ConnectOptions | undefined | null, optio // @public export class Connection { appender(table: string): Promise + arrow(statement: string, params?: Record | null, options?: ZuArrowOptions | null): Promise close(): void columnar(statement: string, params?: Record | null, options?: ZuStatementOptions | null): Promise cursor(statement: string, params?: Record | null, options?: ZuStreamOptions | null): ZuCursor @@ -66,6 +67,7 @@ export function load(path: string, options: ZuLoadOptions): Promise // @public export class Prepared { + arrow(params?: Record | null, options?: ZuArrowOptions | null): Promise close(): Promise get closed(): boolean columnar(params?: Record | null, options?: ZuStatementOptions | null): Promise @@ -101,6 +103,21 @@ export type ZuAppendValue = | ZuDuration | ZuTemporalValue +// @public +export interface ZuArrow { + // (undocumented) + readonly gqlstatus: string + readonly ipc: Uint8Array + // (undocumented) + readonly notices: ZuNotice[] + readonly rows: number +} + +// @public +export interface ZuArrowOptions extends ZuStatementOptions { + readonly batchRows?: number +} + // @public export interface ZuArrowTable { // (undocumented) diff --git a/package.json b/package.json index 094a98b..057af1e 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "reference": "node tools/reference.mjs reference", "bench": "node bench/query.mjs", "bench:append": "node bench/append.mjs", + "bench:arrow": "node bench/arrow.mjs", "bench:columnar": "node bench/columnar.mjs", "bench:load": "node bench/load.mjs", "bench:prepared": "node bench/prepared.mjs", diff --git a/src/arrow.rs b/src/arrow.rs new file mode 100644 index 0000000..c6a010f --- /dev/null +++ b/src/arrow.rs @@ -0,0 +1,107 @@ +//! A result as Arrow, in the bytes Arrow ships between processes. +//! +//! ```js +//! import { tableFromIPC } from 'apache-arrow' +//! const read = await conn.arrow('MATCH (p:person) RETURN p.name AS name, p.age AS age') +//! const table = tableFromIPC(read.ipc) +//! ``` +//! +//! [`columnar`] hands over the buffers themselves and leaves the reader +//! to put a type around them, which is the fastest way out and the one +//! that costs a caller ten lines of Arrow before they have a table. This +//! is the other way: the same buffers, with the schema written beside +//! them, in the format every Arrow implementation already reads. What +//! comes back is bytes, so `apache-arrow` reads it, DuckDB-Wasm reads +//! it, a `fetch` response body carries it, and a worker gets it as a +//! transferable rather than as a structured clone. +//! +//! The translation is not written here. `zu-arrow` in the engine tree is +//! the one answer about what a zu column becomes in Arrow, shared with +//! the Python client, because a second copy of it would be a second set +//! of rules about what a year-month duration is. This module is the +//! runtime's half: read the option, run the statement, hand the bytes to +//! V8 without copying them again. +//! +//! Why bytes and not the C Data Interface, which is what the Python +//! client takes and is a pointer rather than a serialization: nothing in +//! a JavaScript runtime can dereference a pointer. An addon can, and +//! this one does on the way in, but the value that reaches JavaScript +//! has to be something V8 holds, and the only thing V8 holds that Arrow +//! also speaks is a buffer of IPC bytes. The framing is the cost: a +//! schema message, then a header a batch, which is kilobytes against a +//! result of any size and the price of a format with readers. +//! +//! [`columnar`]: crate::columns +use napi::bindgen_prelude::*; +use napi::{Env, ScopedTask}; +use zudb::DiagnosticRecord; + +use crate::conn::{Failure, QueryTask, notices}; + +/// One statement, read as Arrow. +pub struct ArrowTask { + pub(crate) task: QueryTask, + /// How many rows go in a record batch, or why the caller's answer to + /// that could not be read. + pub(crate) batch: std::result::Result, +} + +/// A whole result, as the bytes and what came with them. +pub struct Read { + ipc: Vec, + rows: usize, + gqlstatus: &'static str, + notices: Vec, +} + +impl ArrowTask { + fn run(&mut self) -> std::result::Result { + // Before the statement, because a batch size nobody could read + // is a call that was wrong when it was written and not an answer + // worth running a scan for. + let batch = self + .batch + .as_ref() + .map_err(|message| Failure::Usage(message.clone()))?; + let batch = *batch; + let (result, shape) = self.task.run()?; + // The names are the ones the statement's own catalog gave, so a + // node column names its table rather than its id. + let ipc = zu_arrow::ipc(&result, shape.names(), batch) + .map_err(|err| Failure::Usage(err.to_string()))?; + Ok(Read { + ipc, + // Answered off the columns the sink filled, which is a read + // of a length and not a pass that builds rows nobody wants. + rows: result.rows.len(), + gqlstatus: result.status().code(), + notices: result.notices, + }) + } +} + +impl<'task> ScopedTask<'task> for ArrowTask { + type Output = std::result::Result; + type JsValue = Object<'task>; + + fn compute(&mut self) -> Result { + Ok(self.run()) + } + + fn resolve(&mut self, env: &'task Env, output: Self::Output) -> Result { + let read = output.map_err(|failure| self.task.failed(env, failure))?; + let mut object = Object::new(env)?; + // The `Vec` is moved and not read: the pointer V8 is handed is + // the pointer the writer filled, and the allocation is freed when + // the typed array is collected. + object.set("ipc", Uint8Array::new(read.ipc))?; + object.set("rows", read.rows as f64)?; + object.set("gqlstatus", read.gqlstatus)?; + object.set("notices", notices(env, &read.notices)?)?; + Ok(object) + } + + fn finally(mut self, env: Env) -> Result<()> { + self.task.release(&env) + } +} diff --git a/src/conn.rs b/src/conn.rs index 540c31a..067d3dc 100644 --- a/src/conn.rs +++ b/src/conn.rs @@ -33,6 +33,7 @@ use zudb::query::{QueryResult, Value}; use zudb::{Config, Database, DiagnosticRecord, Interrupt, ZuError}; use crate::append::OpenTask; +use crate::arrow::ArrowTask; use crate::cancel::Watch; use crate::columns::ColumnsTask; use crate::error::{aborted, raise, usage}; @@ -742,6 +743,44 @@ impl Connection { AsyncTask::new(ColumnsTask(self.task(env, statement, params, options))) } + /// Runs one statement and gives back the bytes of an Arrow IPC + /// stream. + /// + /// ```js + /// import { tableFromIPC } from 'apache-arrow' + /// const read = await conn.arrow('MATCH (a:account) RETURN a.name AS name, a.balance AS balance') + /// const table = tableFromIPC(read.ipc) + /// ``` + /// + /// The same buffers [`Connection::columnar`] hands over, with the + /// schema written beside them in the format every Arrow + /// implementation already reads. That is the difference worth + /// knowing: `columnar` is the fastest way out and leaves the caller + /// to say what each buffer means, and this is the one where the + /// result arrives as a table, a dataframe or a DuckDB relation with + /// no code in between. + /// + /// The translation lives in the engine and is the same one the + /// Python client exports through, so a node column names its table + /// in both and a year-month duration is a month interval in both. + #[napi( + ts_args_type = "statement: string, params?: Record | null, options?: ZuArrowOptions | null", + ts_return_type = "Promise" + )] + pub fn arrow( + &self, + env: &Env, + statement: Unknown<'_>, + params: Option>, + options: Option>, + ) -> AsyncTask { + let batch = batch(options.as_ref()); + AsyncTask::new(ArrowTask { + task: self.task(env, statement, params, options), + batch, + }) + } + /// Compiles a statement, pins it, and hands back something that /// runs it. /// @@ -1293,6 +1332,20 @@ fn batch_rows(options: Option<&Object<'_>>) -> std::result::Result, } } +/// The same option, read for a record batch rather than for a stream. +/// +/// Absent is Arrow's own batch, which is what a reader expects and what +/// keeps a working set inside a cache. The arrays are already in memory +/// either way, so a batch here is a slice of them and not a copy, and +/// the size is about what the reader on the other side wants to hold at +/// once. +pub(crate) fn batch(options: Option<&Object<'_>>) -> std::result::Result { + Ok(match batch_rows(options)? { + Some(rows) => rows as usize, + None => zu_arrow::BATCH, + }) +} + /// Reads an option that has to be a boolean, and says what arrived /// instead. /// diff --git a/src/lib.rs b/src/lib.rs index 0c73ad1..8570c9e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,7 @@ //! without. mod append; +mod arrow; mod buffer; mod cancel; mod columns; diff --git a/src/prepared.rs b/src/prepared.rs index fbcabe5..82ebef3 100644 --- a/src/prepared.rs +++ b/src/prepared.rs @@ -48,9 +48,10 @@ use napi::{Env, ScopedTask}; use napi_derive::napi; use zudb::Interrupt; +use crate::arrow::ArrowTask; use crate::columns::ColumnsTask; use crate::conn::{ - CLOSED, ExecTask, Failure, Handles, QueryTask, Source, bind, failed, int_mode, watch, + CLOSED, ExecTask, Failure, Handles, QueryTask, Source, batch, bind, failed, int_mode, watch, wire_disposal, with, }; use crate::value::Spelling; @@ -161,6 +162,24 @@ impl Prepared { AsyncTask::new(ColumnsTask(self.task(env, params, options))) } + /// Runs it and gives back the bytes of an Arrow IPC stream. + #[napi( + ts_args_type = "params?: Record | null, options?: ZuArrowOptions | null", + ts_return_type = "Promise" + )] + pub fn arrow( + &self, + env: &Env, + params: Option>, + options: Option>, + ) -> AsyncTask { + let batch = batch(options.as_ref()); + AsyncTask::new(ArrowTask { + task: self.task(env, params, options), + batch, + }) + } + /// Gives the id back to the session. /// /// Closing twice does nothing the second time, and closing one whose diff --git a/src/value.rs b/src/value.rs index b7da1a1..77a6c86 100644 --- a/src/value.rs +++ b/src/value.rs @@ -103,6 +103,12 @@ impl Shape { spelling, } } + + /// The table names, for the translation into Arrow, which asks for + /// them rather than keeping a copy of its own. + pub fn names(&self) -> &Names { + &self.names + } } /// What the tables in a result are called. @@ -153,6 +159,21 @@ impl Names { } } +/// The same names, borrowed rather than copied, which is what the Arrow +/// translation wants: a column of a million nodes is a million lookups +/// and none of them should allocate. The stand-in for a table the +/// catalog no longer has is written on that side, in the one place both +/// clients share, so it reads the same here and in Python. +impl zu_arrow::Tables for Names { + fn node(&self, id: u32) -> Option<&str> { + self.nodes.get(&id).map(String::as_str) + } + + fn rel(&self, id: u32) -> Option<&str> { + self.rels.get(&id).map(String::as_str) + } +} + /// One node of the graph. /// /// The table is the name written in the schema and the offset is the diff --git a/test/arrow.test.mjs b/test/arrow.test.mjs new file mode 100644 index 0000000..f6ae62f --- /dev/null +++ b/test/arrow.test.mjs @@ -0,0 +1,349 @@ +// A result as Arrow, in the bytes Arrow ships between processes. +// +// The translation lives in the engine and is tested there against the +// arrays it builds. What these check is the half that is this client's: +// that the bytes are a stream `apache-arrow` reads, that what comes out +// of it is what the statement produced, and that a call which cannot +// mean anything is refused with a reason rather than with a buffer. + +import assert from 'node:assert/strict' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { tableFromIPC } from 'apache-arrow' +import { connect, load } from 'zudb' + +import { fresh, isZuError, twoPeople } from './helper.mjs' + +// The table the bytes hold, which is the whole of what a caller writes. +function read(answer) { + return tableFromIPC(answer.ipc) +} + +// One column by name, since a test asks about one of them and the order +// they were projected in is asserted where it is the question. +function column(table, name) { + const at = table.schema.fields.findIndex((field) => field.name === name) + assert.notEqual(at, -1, `no column called ${name}`) + return table.getChildAt(at) +} + +// What a column holds, as plain JavaScript values in row order. +function values(table, name) { + return [...column(table, name)] +} + +// The graph with edges in it, since a load is the only way a JavaScript +// program makes one: three people and the two edges between them. +async function three(t) { + const dir = await mkdtemp(join(tmpdir(), 'zu-node-arrow-')) + t.after(() => rm(dir, { recursive: true, force: true })) + const path = join(dir, 'g.zu1') + await load(path, { + nodes: 'person', + rels: 'knows', + columns: { uid: [10, 20, 30], name: ['ada', 'grace', 'kay'] }, + edges: [ + [0, 1], + [1, 2], + ], + }) + const conn = await connect(path, { readOnly: true }) + t.after(() => conn.close()) + return { conn, path } +} + +test('a result is a stream apache-arrow reads without being told anything', async (t) => { + const { conn } = await twoPeople(t) + const answer = await conn.arrow('MATCH (p:person) RETURN p.id AS id, p.name AS name') + + assert.equal(answer.rows, 2) + assert.ok(answer.ipc instanceof Uint8Array) + const table = read(answer) + assert.equal(table.numRows, 2) + // The schema travels with the bytes, so the reader knows the columns + // and their types without a second call and without a convention. + assert.deepEqual( + table.schema.fields.map((field) => [field.name, String(field.type)]), + [ + ['id', 'Int64'], + ['name', 'Utf8'], + ], + ) + assert.deepEqual(values(table, 'id'), [1n, 2n]) + assert.deepEqual(values(table, 'name'), ['ada', 'zoe']) +}) + +test('the columns come back in the order the statement projected them', async (t) => { + const { conn } = await twoPeople(t) + const table = read(await conn.arrow('MATCH (p:person) RETURN p.name AS name, p.id AS id')) + assert.deepEqual(table.schema.fields.map((field) => field.name), ['name', 'id']) +}) + +test('floats and booleans are the Arrow types they are', async (t) => { + const { conn } = await fresh(t) + await conn.exec('INSERT (m:measure {id: 1, ratio: 1.5, hot: true})') + await conn.exec('INSERT (m:measure {id: 2, ratio: -0.25, hot: false})') + const table = read(await conn.arrow('MATCH (m:measure) RETURN m.ratio AS ratio, m.hot AS hot')) + + assert.equal(String(column(table, 'ratio').type), 'Float64') + assert.deepEqual(values(table, 'ratio'), [1.5, -0.25]) + assert.equal(String(column(table, 'hot').type), 'Bool') + assert.deepEqual(values(table, 'hot'), [true, false]) +}) + +test('a temporal column is the Arrow type that counts the same thing', async (t) => { + const { conn } = await fresh(t) + await conn.exec( + "INSERT (e:event {id: 1, on: DATE '2024-01-01', at: LOCAL DATETIME '2024-01-02T03:04:05', " + + "took: DURATION 'PT1H'})", + ) + const table = read( + await conn.arrow('MATCH (e:event) RETURN e.on AS on, e.at AS at, e.took AS took'), + ) + + // A date is days, a datetime is nanoseconds, and a day-time duration + // is a duration rather than an interval, which is the difference + // between a length of time and a calendar step. + assert.equal(String(column(table, 'on').type), 'Date32') + assert.equal(String(column(table, 'at').type), 'Timestamp') + assert.equal(String(column(table, 'took').type), 'Duration') +}) + +test('a zoned datetime carries its zone in the type', async (t) => { + const { conn } = await fresh(t) + const table = read( + await conn.arrow("RETURN ZONED DATETIME '2024-01-02T03:04:05+02:00' AS d"), + ) + // Arrow says the zone on the field rather than on the value, and an + // offset is a zone Arrow accepts as one. + assert.match(String(column(table, 'd').type), /\+02:00/) +}) + +test('a year-month duration is a month interval and not a count of nanoseconds', async (t) => { + const { conn } = await fresh(t) + const table = read(await conn.arrow("RETURN DURATION 'P14M' AS every")) + assert.match(String(column(table, 'every').type), /Interval/) +}) + +test('a null row is a null and not a zero', async (t) => { + const { conn } = await twoPeople(t) + const table = read( + await conn.arrow('MATCH (p:person) RETURN CASE WHEN p.id = 1 THEN p.id ELSE null END AS maybe'), + ) + assert.deepEqual(values(table, 'maybe'), [1n, null]) + assert.equal(column(table, 'maybe').nullCount, 1) +}) + +test('a node column is a struct naming the table it came from', async (t) => { + const { conn } = await twoPeople(t) + const table = read(await conn.arrow('MATCH (p:person) RETURN p AS who')) + + const who = column(table, 'who') + assert.match(String(who.type), /Struct/) + assert.deepEqual( + [...who].map((node) => [node.table, node.offset]), + [ + ['person', 0n], + ['person', 1n], + ], + ) +}) + +test('an edge column is a struct with the ends it joins', async (t) => { + const { conn } = await three(t) + const table = read(await conn.arrow('MATCH ()-[r:knows]->() RETURN r AS r')) + + assert.deepEqual( + [...column(table, 'r')].map((rel) => [rel.table, rel.src, rel.dst, rel.ord]), + [ + ['knows', 0n, 1n, 0n], + ['knows', 1n, 2n, 1n], + ], + ) +}) + +test('a path is the nodes it walked and the edges it crossed', async (t) => { + const { conn } = await three(t) + const table = read( + await conn.arrow('MATCH q = (a:person)-[:knows]->()-[:knows]->(c:person) RETURN q AS q'), + ) + + // Two lists rather than one alternating list, because Arrow has no + // type for a list whose elements change shape, and a walk is one more + // node than edge either way. + const [walk] = [...column(table, 'q')] + assert.deepEqual([...walk.nodes].map((node) => node.offset), [0n, 1n, 2n]) + assert.deepEqual( + [...walk.rels].map((rel) => [rel.src, rel.dst]), + [ + [0n, 1n], + [1n, 2n], + ], + ) +}) + +test('a list is an Arrow list and a record is an Arrow struct', async (t) => { + const { conn } = await twoPeople(t) + const table = read( + await conn.arrow('MATCH (p:person) RETURN [p.id, p.id] AS pair, {name: p.name} AS held'), + ) + + assert.match(String(column(table, 'pair').type), /List/) + assert.deepEqual([...[...column(table, 'pair')][0]], [1n, 1n]) + assert.equal([...column(table, 'held')][1].name, 'zoe') +}) + +test('a statement that matched nothing still says what its columns were', async (t) => { + const { conn } = await twoPeople(t) + const answer = await conn.arrow( + "MATCH (p:person) WHERE p.name = 'nobody' RETURN p.id AS id, p.name AS name", + ) + + assert.equal(answer.rows, 0) + const table = read(answer) + assert.equal(table.numRows, 0) + // The plan declared the types, so an empty answer has the schema a + // full one would have had, which is what makes a table built from one + // concatenable with a table built from the other. + assert.deepEqual( + table.schema.fields.map((field) => String(field.type)), + ['Int64', 'Utf8'], + ) +}) + +test('a statement that projects nothing has no columns and says so', async (t) => { + const { conn } = await fresh(t) + const answer = await conn.arrow("INSERT (p:person {id: 1, name: 'ada'})") + + assert.equal(answer.rows, 0) + assert.equal(read(answer).schema.fields.length, 0) + // 00001 is the standard's own way of saying the statement completed + // and had no result to give back. + assert.equal(answer.gqlstatus, '00001') +}) + +test('the status and the notices ride beside the bytes', async (t) => { + const { conn } = await twoPeople(t) + const answer = await conn.arrow('MATCH (p:person) RETURN p.id AS id') + assert.equal(answer.gqlstatus, '00000') + assert.deepEqual(answer.notices, []) +}) + +test('parameters bind the same way they do for rows', async (t) => { + const { conn } = await twoPeople(t) + const table = read( + await conn.arrow('MATCH (p:person) WHERE p.name = $name RETURN p.id AS id', { name: 'zoe' }), + ) + assert.deepEqual(values(table, 'id'), [2n]) +}) + +test('batchRows cuts the stream into the batches a caller asked for', async (t) => { + const { conn } = await twoPeople(t) + const table = read(await conn.arrow('MATCH (p:person) RETURN p.id AS id', null, { batchRows: 1 })) + + assert.equal(table.batches.length, 2) + assert.deepEqual(table.batches.map((batch) => batch.numRows), [1, 1]) + // Cutting changes nothing about the answer, which is the point: the + // arrays are built whole and a batch is a slice of them. + assert.deepEqual(values(table, 'id'), [1n, 2n]) +}) + +test('a batch size that could never hold a row is refused', async (t) => { + const { conn } = await twoPeople(t) + const caught = await conn + .arrow('MATCH (p:person) RETURN p.id AS id', null, { batchRows: 0 }) + .then(() => null, (err) => err) + + assert.ok(isZuError(caught, 'ZuUsageError')) + assert.match(caught.message, /batchRows is 0/) +}) + +test('a column that mixes two types is refused, with the row that did it', async (t) => { + const { conn } = await twoPeople(t) + const caught = await conn + .arrow('MATCH (p:person) RETURN CASE WHEN p.id = 1 THEN p.id ELSE p.name END AS mixed') + .then(() => null, (err) => err) + + assert.ok(isZuError(caught, 'ZuUsageError')) + assert.match(caught.message, /column 'mixed' mixes integers and strings at row 1/) +}) + +test('a type Arrow has nowhere to put is refused by name', async (t) => { + const { conn } = await fresh(t) + const caught = await conn + .arrow("RETURN ZONED TIME '03:04:05+02:00' AS t") + .then(() => null, (err) => err) + + // Arrow has a time and a timestamp and nothing in between, and + // dropping the offset would move the value. The columnar read hands + // this one over as cells and an offset beside them, which is the way + // out for a caller who wants it. + assert.ok(isZuError(caught, 'ZuUsageError')) + assert.match(caught.message, /time with an offset, which Arrow has no type for/) +}) + +test('a statement can be stopped by a signal like any other', async (t) => { + const { conn } = await twoPeople(t) + const caught = await conn + .arrow('MATCH (p:person) RETURN p.id AS id', null, { signal: AbortSignal.abort() }) + .then(() => null, (err) => err) + + assert.equal(caught.name, 'AbortError') + assert.equal((await conn.arrow('MATCH (p:person) RETURN p.id AS id')).rows, 2) +}) + +test('a closed connection refuses the call as a rejection', async (t) => { + const { conn } = await twoPeople(t) + await conn.close() + const caught = await conn + .arrow('MATCH (p:person) RETURN p.id AS id') + .then(() => null, (err) => err) + assert.ok(isZuError(caught, 'ZuUsageError')) +}) + +test('a prepared statement reads as Arrow too', async (t) => { + const { conn } = await twoPeople(t) + await using find = await conn.prepare('MATCH (p:person) WHERE p.name = $name RETURN p.id AS id') + const table = read(await find.arrow({ name: 'zoe' })) + assert.deepEqual(values(table, 'id'), [2n]) +}) + +test('the bytes are handed over rather than shared, so two reads are two buffers', async (t) => { + const { conn } = await twoPeople(t) + const first = await conn.arrow('MATCH (p:person) RETURN p.id AS id') + const second = await conn.arrow('MATCH (p:person) RETURN p.id AS id') + + first.ipc[0] = 0xff + // Writing into one is writing into a buffer nothing else is reading, + // which is what makes handing the memory over safe. + assert.deepEqual(values(read(second), 'id'), [1n, 2n]) +}) + +test('a million rows are one stream and the loop stays free while it is written', async (t) => { + const { conn } = await fresh(t) + await conn.exec('INSERT (n:number {id: 1, at: 1})') + const rows = 1_000_000 + const appender = await conn.appender('number') + for (let at = 2; at <= rows; at += 1) appender.appendRow([BigInt(at), BigInt(at)]) + await appender.close() + + let ticks = 0 + const timer = setInterval(() => (ticks += 1), 1) + const at = performance.now() + const answer = await conn.arrow('MATCH (n:number) RETURN n.at AS at') + const took = performance.now() - at + clearInterval(timer) + + assert.equal(answer.rows, rows) + const table = read(answer) + assert.equal(table.numRows, rows) + assert.equal(column(table, 'at').get(rows - 1), BigInt(rows)) + // The whole write is on the threadpool, so the timer kept firing + // throughout it rather than queueing behind it. The bar is a tick + // every ten milliseconds of the read and not a fixed count, because a + // blocked loop fires none however long the read takes. + assert.ok(ticks > took / 10, `the event loop ticked ${ticks} times in ${took.toFixed(0)} ms`) +}) diff --git a/types/header.d.ts b/types/header.d.ts index 56f34d0..ccd123a 100644 --- a/types/header.d.ts +++ b/types/header.d.ts @@ -377,6 +377,46 @@ export interface ZuColumnar { readonly notices: ZuNotice[] } +/** + * A whole result as Arrow, in the bytes Arrow ships between processes. + * + * The same buffers a columnar read hands over, with the schema written + * beside them, so `tableFromIPC(read.ipc)` is the whole of the reading + * code and every Arrow implementation is a reader. A result with no rows + * is a schema and one empty batch rather than nothing at all, so the + * columns are known either way. + */ +export interface ZuArrow { + /** + * The stream, as one buffer: a schema message and then a message a + * batch. It is the addon's own allocation handed over rather than + * copied, and it detaches when posted to a worker, which is what makes + * a result cross a thread without being cloned. + */ + readonly ipc: Uint8Array + /** How many rows are in it, which the batches also add up to. */ + readonly rows: number + readonly gqlstatus: string + readonly notices: ZuNotice[] +} + +/** + * What a statement read as Arrow takes beside its parameters. + */ +export interface ZuArrowOptions extends ZuStatementOptions { + /** + * How many rows one record batch holds. Arrow's own 65,536 by + * default, which is what a reader expects and what keeps a batch + * inside a cache. + * + * The arrays are built whole either way and a batch is a slice of + * them, so this costs nothing to change and buys nothing to tune. It + * is worth naming when the reader on the other side has a size of its + * own, or when the batches are going somewhere one at a time. + */ + readonly batchRows?: number +} + /** * The rows a statement gave back. *