From 1356a18bab3fd19bb3781937133b277f2df5df0c Mon Sep 17 00:00:00 2001 From: hmirin Date: Wed, 14 Oct 2026 14:31:00 +0900 Subject: [PATCH] Speed up the carrier backend and add a reproducible benchmark - Store carrier ciphertext without compression or deltas and read it with `cat-file --batch`. - Stage objects into the cache before publishing and avoid materializing carrier history in helper checkouts. - Advance the verified frontier after the client's own publication, so the first tiny push after a large one no longer walks the whole history. - Use chacha20poly1305 0.11 with aead-stream, which picks NEON on aarch64 automatically. - Add opt-in helper phase tracing. - Add scripts/benchmark-git-backend.sh and scripts/reproduce-benchmark.sh, which run the full comparison end to end on macOS and Linux, and record the results in docs/benchmarks.md. --- Cargo.lock | 227 ++++++++-- Cargo.toml | 5 +- docs/benchmarks.md | 372 ++++++++++++++- docs/design.md | 44 +- scripts/benchmark-git-backend.sh | 756 +++++++++++++++++++++++++++++++ scripts/reproduce-benchmark.sh | 579 +++++++++++++++++++++++ src/crypto.rs | 105 ++++- src/git.rs | 119 ++++- src/lib.rs | 1 + src/repository.rs | 35 +- src/storage.rs | 607 +++++++++++++++++++++---- src/trace.rs | 110 +++++ tests/carrier_git.rs | 11 +- tests/end_to_end.rs | 125 +++++ 14 files changed, 2943 insertions(+), 153 deletions(-) create mode 100755 scripts/benchmark-git-backend.sh create mode 100755 scripts/reproduce-benchmark.sh create mode 100644 src/trace.rs diff --git a/Cargo.lock b/Cargo.lock index bdb2df1..8aac434 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,10 +8,29 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.2", +] + +[[package]] +name = "aead-stream" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a901e5bcd15b4a9555a8f17a608d28ef92cf77bc4aa3b4a0c1a3fce1a9cc0bac" +dependencies = [ + "aead 0.6.1", +] + [[package]] name = "aes" version = "0.8.4" @@ -19,8 +38,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", - "cpufeatures", + "cipher 0.4.4", + "cpufeatures 0.2.17", ] [[package]] @@ -29,9 +48,9 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ - "aead", + "aead 0.5.2", "aes", - "cipher", + "cipher 0.4.4", "ctr", "ghash", "subtle", @@ -120,6 +139,25 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -133,8 +171,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", - "cipher", - "cpufeatures", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cipher 0.5.2", + "cpufeatures 0.3.1", ] [[package]] @@ -143,24 +192,47 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ - "aead", - "chacha20", - "cipher", - "poly1305", + "aead 0.5.2", + "chacha20 0.9.1", + "cipher 0.4.4", + "poly1305 0.8.0", "zeroize", ] +[[package]] +name = "chacha20poly1305" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" +dependencies = [ + "aead 0.6.1", + "chacha20 0.10.2", + "cipher 0.5.2", + "poly1305 0.9.1", +] + [[package]] name = "cipher" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", - "inout", + "crypto-common 0.1.7", + "inout 0.1.4", "zeroize", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.2", +] + [[package]] name = "clap" version = "4.6.6" @@ -201,6 +273,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -222,6 +300,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -233,13 +320,33 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + [[package]] name = "ctr" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", ] [[package]] @@ -249,7 +356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -285,8 +392,8 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] @@ -337,6 +444,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + [[package]] name = "fs2" version = "0.4.3" @@ -389,6 +502,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -405,9 +519,10 @@ dependencies = [ name = "git-remote-e2ee" version = "0.1.0" dependencies = [ + "aead-stream", "anyhow", "base64", - "chacha20poly1305", + "chacha20poly1305 0.11.0", "clap", "ed25519-dalek", "fs2", @@ -462,9 +577,9 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f65d16b699dd1a1fa2d851c970b0c971b388eeeb40f744252b8de48860980c8f" dependencies = [ - "aead", + "aead 0.5.2", "aes-gcm", - "chacha20poly1305", + "chacha20poly1305 0.10.1", "digest", "generic-array", "hkdf", @@ -476,6 +591,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "inout" version = "0.1.4" @@ -485,6 +609,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -549,9 +682,19 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", +] + +[[package]] +name = "poly1305" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" +dependencies = [ + "cpufeatures 0.3.1", + "universal-hash 0.6.1", ] [[package]] @@ -561,9 +704,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", ] [[package]] @@ -664,6 +807,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rustc_version" version = "0.4.1" @@ -742,10 +891,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", + "sha2-asm", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signature" version = "2.2.0" @@ -850,10 +1015,20 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "utf8parse" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index edac990..e4a59e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,8 @@ path = "src/bin/git-remote-e2ee.rs" [dependencies] anyhow = "1.0" base64 = "0.22" -chacha20poly1305 = { version = "0.10", features = ["stream"] } +aead-stream = "0.6" +chacha20poly1305 = "0.11" clap = { version = "4.5", features = ["derive"] } ed25519-dalek = { version = "2.1", features = ["rand_core"] } fs2 = "0.4" @@ -33,7 +34,7 @@ rand_09 = { package = "rand", version = "0.9", features = ["os_rng", "std_rng"], rand_chacha = "0.9" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sha2 = "0.10" +sha2 = { version = "0.10", features = ["asm"] } subtle = "2.6" thiserror = "2.0" tempfile = "3.14" diff --git a/docs/benchmarks.md b/docs/benchmarks.md index d7fd7f4..d2b4dfd 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,4 +1,365 @@ -# Local performance benchmarks +# Benchmarks + +## Reproduce + +From a fresh clone on macOS or Linux, run: + +```sh +./scripts/reproduce-benchmark.sh +``` + +The first run downloads about 1 GB for Godot's full default-branch history and +the pinned gcrypt source. Keep free temporary disk space of about 12 times the +Godot `.git` size. The default three-round run takes hours. Results go to +`./bench-results/-/` and include raw and median TSV files, +`environment.json`, and a README-format `summary.md`. + +On Linux, the work directory defaults to `/var/tmp` because `/tmp` may be +tmpfs. The runner detects the selected filesystem and refuses tmpfs or ramfs. +Set `TMPDIR` to a disk-backed temporary directory, or set +`BENCH_WORK_PARENT` to a disk-backed path under `/var/tmp` or `TMPDIR`. + +The fresh-fetch client-disk row totals the client .git directory and any +auxiliary cache or state. + +## Latest reproduce run (2026-10-08) + +The README table comes from this run of `scripts/reproduce-benchmark.sh` with +its defaults (three rounds, five tiny commits), at e2ee `cedc9fe` with no +uncommitted changes, on an Apple M1 Pro (10 cores) with 32 GiB RAM, +macOS 26.5.2, Git 2.55.0, and Homebrew Rust 1.98.0. The machine was otherwise +lightly loaded. The runner's `summary.md`: + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial encryption (to a local directory) | 28.6 s | 11.0 s | 8.1 s | +| Initial push | 28.4 s | 36.6 s | 22.9 s | +| Fresh fetch | 27.7 s | 55.0 s | 40.0 s | +| Tiny commit | 0.09 s | 0.09 s | 0.09 s | +| Tiny push | 0.08 s | 6.5 s | 0.97 s | +| Tiny update (fetch) | 0.07 s | 22.6 s | 0.77 s | +| Data sent per tiny push | 4.5 KB | 921 MB | 8.9 KB | +| Peak memory, initial push | 1.27 GiB | 0.98 GiB | 1.27 GiB | +| Peak memory, tiny push | 37 MiB | 887 MiB | 9 MiB | +| Remote size after 5 tiny pushes | 891 MiB | 5,272 MiB | 878 MiB | +| Total client disk after fresh fetch (.git + auxiliary cache/state) | 891 MiB | 1,780 MiB | 1,779 MiB | + +### Second macOS machine (2026-10-08) + +The same runner at e2ee `971894a` on an Apple M1 Max with 64 GiB RAM, +macOS 27.0, and Git 2.56.0. That machine was shared with other work during the +run (load average about 7-16 on 10 cores). The ordering is identical: + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial encryption (to a local directory) | 30.3 s | 12.0 s | 8.9 s | +| Initial push | 30.3 s | 37.6 s | 23.9 s | +| Fresh fetch | 28.0 s | 53.5 s | 38.0 s | +| Tiny commit | 0.16 s | 0.16 s | 0.16 s | +| Tiny push | 0.09 s | 6.8 s | 1.2 s | +| Tiny update (fetch) | 0.07 s | 24.3 s | 0.84 s | +| Data sent per tiny push | 4.8 KB | 921 MB | 9.5 KB | +| Peak memory, initial push | 1.27 GiB | 0.96 GiB | 1.27 GiB | +| Peak memory, tiny push | 37 MiB | 887 MiB | 9 MiB | +| Remote size after 5 tiny pushes | 891 MiB | 5,272 MiB | 878 MiB | +| Total client disk after fresh fetch (.git + auxiliary cache/state) | 891 MiB | 1,780 MiB | 1,779 MiB | + +The sections below record earlier runs and the investigation that led here. + +## Primary Git-backend benchmark: Godot (2026-10-03) + +This compares plain Git over `file://` with `--no-local`, `git-remote-gcrypt` +using its Git backend (`gcrypt::file://...`), and E2EE using its carrier-Git +backend (`e2ee::git+file://...`). Initial encryption is a separate local +backend baseline: plain Git pushes to a local bare repository, while gcrypt +and E2EE use their local-directory backends. + +The source was `godotengine/godot` at `00932449c9f372b30301d8b5fdc1be70ec12b5c0` +(default branch only, no tags): 85,678 commits, 14,162 files at HEAD, and +910,290,704 reachable object bytes. The gcrypt helper was at +`a5ff704d071f14b95b6b1fa0caa8cdbf0c6cdadb`; the E2EE release binary was built +from `7036a37dda4eea16195e19429e5a5807ebd37858`. Each of three rounds used +fresh bare remotes, clients, cache directories, and a temporary GPG home. Git +auto-GC, maintenance, and receive auto-GC were disabled. The bare receivers +used `receive.unpackLimit=0`; the carrier receiver also used +`attr.tree=refs/heads/git-remote-e2ee`, so it could read the committed +`e2ee/** -delta` attribute. The E2EE push and returning-client caches were +isolated per round under the temporary benchmark directory. Each tiny commit +replaced one line in Godot's tracked README and was pushed and fetched before +the next. See `scripts/benchmark-git-backend.sh` for the complete harness. + +Machine: Apple M1 Max (10 CPU cores), 64 GiB RAM (`sysctl hw.memsize` = +68,719,476,736 bytes), macOS 27.0 build 26A428. Versions: Git 2.55.0, GnuPG +2.5.24, Rust 1.97.1. + +The harness wraps every measured command in macOS `/usr/bin/time -l` (or GNU +`/usr/bin/time -v` on Linux). Peak RSS is the platform-reported maximum for +that command and its waited-for children, where supported; it is not the sum +of concurrent process RSS. Logical directory size sums regular-file lengths. +Allocated size uses `du -sk` in 1 KiB blocks. `Remote / stored L/A` is logical +and allocated MiB for the local storage directory or bare remote after the +phase. `Client .git L/A` is the whole client Git directory after the phase. +`Auxiliary state / cache L/A` is gcrypt's `.git/remote-gcrypt` state or the +whole isolated E2EE cache; `—` means not applicable. The byte column is the +logical object-store increase for pushes and tiny updates, the stored amount +for initial encryption, and the remote object-store reference size for fresh +fetch. Fetch rows also include client object-store growth in the final column. +These are filesystem/object-database measurements, not packet-level wire +counts. Gcrypt's client `.git` contains both encrypted carrier objects and +imported inner Git objects. The sizes are workload-specific and the local +`file://` timings do not predict GitHub server throughput. +The harness processes used only run-specific caches under the temporary +benchmark directory. A separate short-lived Git helper against the default +user cache appeared in process listings during the run; it was left untouched, +and any timing effect from that concurrent activity is unquantified. + +Medians across three fresh rounds; repository sizes are MiB (`L/A` = logical / +allocated). The byte column remains exact bytes. + +| Phase | Transport | Time | Peak RSS | Object/stored bytes | Remote / stored L/A | Client `.git` L/A | Auxiliary state / cache L/A | Client object growth B | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| Initial encryption | Plain Git | 31.97 s | 1302.70 MiB | 934,460,946 | 891.20 / 903.80 | 892.77 / 904.83 | — | — | +| Initial encryption | gcrypt | 13.38 s | 983.23 MiB | 921,021,599 | 878.35 / 878.36 | 892.77 / 904.84 | — | — | +| Initial encryption | E2EE | 10.08 s | 1302.02 MiB | 920,636,512 | 877.99 / 880.02 | 892.77 / 904.85 | — | — | +| Initial push | Plain Git | 32.91 s | 1301.84 MiB | 934,460,946 | 891.20 / 903.42 | 892.77 / 904.86 | — | — | +| Initial push | gcrypt | 41.11 s | 983.28 MiB | 921,304,151 | 878.65 / 881.59 | 1771.39 / 1785.52 | — | — | +| Initial push | E2EE | 25.76 s | 1304.09 MiB | 920,777,562 | 878.15 / 880.31 | 1771.39 / 1785.53 | 878.18 / 892.33 | — | +| Fresh fetch | Plain Git | 30.83 s | 1306.02 MiB | 934,460,946 | 891.20 / 903.42 | 891.20 / 903.25 | — | 934,460,946 | +| Fresh fetch | gcrypt | 56.70 s | 886.61 MiB | 921,304,151 | 878.65 / 881.59 | 1779.70 / 1784.19 | 0.00 / 0.00 | 1,866,126,563 | +| Fresh fetch | E2EE | 41.09 s | 886.42 MiB | 920,764,311 | 878.15 / 880.31 | 901.08 / 903.70 | 878.14 / 878.37 | 944,824,608 | +| Tiny commit | Plain Git | 0.19 s | 13.53 MiB | 0 | 891.21 / 903.44 | 1771.41 / 1785.24 | — | — | +| Tiny commit | gcrypt | 0.19 s | 13.53 MiB | 0 | 2635.91 / 2656.59 | 1771.41 / 1785.24 | — | — | +| Tiny commit | E2EE | 0.19 s | 13.53 MiB | 0 | 878.17 / 880.34 | 1771.41 / 1785.24 | 878.20 / 892.42 | — | +| Tiny push (median) | Plain Git | 0.10 s | 37.47 MiB | 4,762 | 891.21 / 903.45 | 1771.41 / 1785.24 | — | — | +| Tiny push (median) | gcrypt | 7.29 s | 887.44 MiB | 921,315,032 | 3514.54 / 3548.37 | 1771.42 / 1785.27 | — | — | +| Tiny push (median) | E2EE | 1.47 s | 37.91 MiB | 9,462 | 878.18 / 880.36 | 1771.42 / 1785.27 | 878.21 / 892.47 | — | +| Tiny update (median) | Plain Git | 0.08 s | 37.33 MiB | 4,762 | 891.21 / 903.45 | 891.21 / 903.30 | — | 3,359 | +| Tiny update (median) | gcrypt | 25.41 s | 886.67 MiB | 921,315,032 | 3514.54 / 3548.37 | 1779.73 / 1784.28 | 0.00 / 0.00 | 9,472 | +| Tiny update (median) | E2EE | 1.04 s | 39.03 MiB | 9,462 | 878.18 / 880.36 | 901.10 / 903.74 | 878.16 / 878.51 | 4,499 | + +The size pairs are rounded to 0.01 MiB. Gcrypt's auxiliary state was 78 B +logical / 4,096 B allocated after fresh fetch and 312 B / 4,096 B after tiny +update; both display as `0.00 / 0.00 MiB` above. Compared with the prior +published run, E2EE's warm tiny-push and tiny-update medians moved from 1.23 s +/ 0.89 s to 1.47 s / 1.04 s. This is an observed difference across separate +three-round runs; the independent helper activity described above overlapped +the latest run, so its cause is unmeasured. + +The per-push medians below include wall time, peak RSS, and logical bytes +added to the bare remote (for fetches, remote bytes associated with that +update followed by client object-store growth). RSS is MiB. + +| Tiny push | Plain Git: time / RSS / remote bytes | gcrypt Git: time / RSS / remote bytes | E2EE carrier Git: time / RSS / remote bytes | +|---|---:|---:|---:| +| 1 (first) | 0.20 s / 37.3 MiB / 4,776 | 8.06 s / 887.1 MiB / 921,307,778 | 5.20 s / 522.3 MiB / 9,345 | +| 2 | 0.10 s / 37.9 MiB / 4,760 | 7.38 s / 887.1 MiB / 921,311,401 | 1.32 s / 38.0 MiB / 9,405 | +| 3 | 0.10 s / 37.9 MiB / 4,762 | 7.21 s / 887.1 MiB / 921,315,032 | 1.35 s / 37.9 MiB / 9,462 | +| 4 | 0.10 s / 37.9 MiB / 4,763 | 7.47 s / 887.2 MiB / 921,318,660 | 1.47 s / 37.9 MiB / 9,522 | +| 5 | 0.10 s / 38.0 MiB / 4,759 | 7.27 s / 887.2 MiB / 921,322,283 | 1.53 s / 37.9 MiB / 9,577 | + +| Tiny update fetch | Plain Git: time / RSS / remote bytes / client growth | gcrypt Git: time / RSS / remote bytes / client growth | E2EE carrier Git: time / RSS / remote bytes / client growth | +|---|---:|---:|---:| +| 1 (first) | 0.18 s / 37.4 MiB / 4,776 / 3,358 B | 25.95 s / 887.0 MiB / 921,307,778 / 9,010 B | 0.89 s / 39.0 MiB / 9,345 / 4,498 B | +| 2 | 0.08 s / 37.3 MiB / 4,760 / 3,359 B | 25.20 s / 887.0 MiB / 921,311,401 / 9,241 B | 0.92 s / 39.0 MiB / 9,405 / 4,500 B | +| 3 | 0.08 s / 37.3 MiB / 4,762 / 3,359 B | 25.33 s / 887.1 MiB / 921,315,032 / 9,472 B | 1.04 s / 39.1 MiB / 9,462 / 4,499 B | +| 4 | 0.09 s / 37.3 MiB / 4,763 / 3,359 B | 25.50 s / 887.1 MiB / 921,318,660 / 9,706 B | 1.04 s / 39.0 MiB / 9,522 / 4,501 B | +| 5 | 0.08 s / 37.3 MiB / 4,759 / 3,356 B | 25.41 s / 887.1 MiB / 921,322,283 / 9,927 B | 1.15 s / 39.1 MiB / 9,577 / 4,498 B | + +In the 2026-10-03 run, gcrypt added about 921 MiB to the remote on each tiny +push. E2EE's first tiny push took 5.20 s and used 522.3 MiB peak RSS. Later +pushes took 1.32 to 1.53 s, used about 38 MiB, and added 9.3 to 9.6 KiB. +E2EE's median tiny push and fetch were below 1.5 s and 1.1 s, respectively. + +#### First-push follow-up (2026-10-08) + +Commit `f4526d8` advances the verified refs after a local publication only +when the client's previous refs already match its verified set and the local +repository is neither shallow nor promisor-enabled. This removes the full +history connectivity walk from the first tiny push after an initial push. + +Both samples used the pinned Godot history, one round, and three tiny commits. +The before sample ran E2EE only. The after sample ran all three backends. + +| Tiny push | Before fix time | Before fix peak RSS | After fix time | After fix peak RSS | +|---|---:|---:|---:|---:| +| 1 | 4.33 s | 528.0 MiB | 1.10 s | 8.97 MiB | +| 2 | 1.08 s | 37.7 MiB | 1.17 s | 8.84 MiB | +| 3 | 1.14 s | 37.7 MiB | 1.32 s | 8.81 MiB | + +A separate pre-fix trace spent 3,335 ms in `git_ref_connectivity_walk` on +push 1. A post-fix trace checked zero objects in that walk and spent 71 ms +there. The Mac's one-minute load average was 11.87 on 10 cores before the +after run. These are single-round follow-up samples, not replacements for the +three-round table above. + +#### Initial-encryption profiling and changes + +The before and after profiles are traced single runs; the final result above +is the median of three untraced rounds. Phase timers overlap and must not be +summed. Before enabling accelerated crypto, the traced directory encryption +took 17.47 s: `git_pack_objects_stream_lifetime` 16.928 s, pack-pipe reads +8.276 s, ciphertext stage writes 3.254 s, SHA-256 2.919 s, ChaCha20-Poly1305 +2.451 s, and durable stage finish 0.033 s. With `sha2` hardware assembly and +the AArch64 ChaCha20 NEON backend, a traced run took 9.80 s: stream lifetime +9.399 s, pipe reads 6.411 s, stage writes 0.877 s, SHA-256 0.453 s, ChaCha +1.657 s, and stage finish 0.033 s. The final untraced median was 10.08 s. +Compared with the prior 14.20 s median, this is 4.12 s (29%) faster; in the +same final run, it beat gcrypt's 13.38 s by 3.30 s (25%). +E2EE used 1302.02 MiB peak RSS for this phase, 318.79 MiB more than gcrypt's +983.23 MiB; Git's inner `pack-objects` process dominates that peak. + +The source repository had no bitmap and no pack-related compression overrides. +The carrier's compression-free settings do not apply to the user's inner Git +repository. A cold standalone `pack-objects --stdout --revs` took 8.96 s. +For gcrypt's shape, `rev-list --objects` took 3.16 s and the following +`pack-objects --stdout` took 4.17 s; the intermediate object list was 46.5 MB +for 756,518 objects. On warm probes, the direct pipeline took 6.33 s versus +6.72 s for `pack-objects --stdout --revs`. Individual options also showed no +stable gain: `--delta-base-offset` took 6.40 s, `--threads=0` 6.37 s, and +`--path-walk` 6.56 s. There was no bitmap index for `--use-bitmap-index` to +use. Existing packed-object reuse remains enabled. A bounded writer-thread +experiment was reverted: its queue wait was only about 40 ms and the traced +end-to-end samples did not improve. +The retained change accelerates the existing SHA-256 and ChaCha20 operations; +the wire format, AAD, segment counters, durability steps, and Git pack +generation command are unchanged. + +#### End-user `cargo install` verification (2026-10-03) + +I built the pushed `64ae6e4` revision using `cargo install --git +https://github.com/combinatrix-ai/git-remote-e2ee --branch readme-rewrite` +from outside the checkout, with an isolated temporary `CARGO_HOME` and no +`RUSTFLAGS`. The exact invocation also used `--root --force +git-remote-e2ee` to isolate the installation. Cargo resolved +`chacha20poly1305 0.11.0`, `chacha20 0.10.2`, and `aead-stream 0.6.0`. A +one-round E2EE-only run of the benchmark harness measured these +initial-encryption values; the other phases from that round are omitted here. + +| Build | Phase | Time | Peak RSS | Stored logical bytes | Stored allocated size | +|---|---|---:|---:|---:|---:| +| `cargo install --git`, no build flags | Initial encryption | 10.080 s | 1304.61 MiB | 920,636,512 B (877.99 MiB) | 880.02 MiB | + +The row is a single run, not a median. Peak RSS is 1,367,982,080 bytes. Stored +allocated size is 922,767,360 bytes. The no-flags AArch64 build selected the +NEON backend; a stream test with tracing reported +`chacha20_backend=aarch64-neon`. + +### Linux aarch64 cross-check (2026-10-06) + +The same `scripts/reproduce-benchmark.sh` was run from a fresh clone, with an +empty input cache, in a Debian 13 LXC container on an Oracle Cloud +`VM.Standard.A1.Flex` host (Ampere Altra, AArch64), limited to 2 CPUs and +6 GiB RAM, on Btrfs. Git 2.47.3, GnuPG 2.4.7, Rust 1.99.0, e2ee `81e43c1` +(the runner revision; protocol and crypto code identical to the macOS run). +Three rounds, five tiny commits each. `TMPDIR=/var/tmp` was set by hand +because that runner revision still defaulted to the container's tmpfs `/tmp`; +later revisions default to `/var/tmp` on Linux. + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial encryption (to a local directory) | 82.7 s | 22.8 s | 24.2 s | +| Initial push | 81.1 s | 82.8 s | 64.1 s | +| Fresh fetch | 92.4 s | 145.7 s | 118.9 s | +| Tiny commit | 0.07 s | 0.07 s | 0.07 s | +| Tiny push | 0.04 s | 23.9 s | 0.22 s | +| Tiny update (fetch) | 0.03 s | 43.4 s | 0.20 s | +| Data sent per tiny push | 4.5 KB | 921 MB | 8.9 KB | +| Peak memory, initial push | 1.11 GiB | 0.96 GiB | 1.11 GiB | +| Peak memory, tiny push | 44 MiB | 884 MiB | 48 MiB | +| Remote size after 5 tiny pushes | 891 MiB | 5,272 MiB | 878 MiB | +| Total client disk after fresh fetch (.git + auxiliary cache/state) | 891 MiB | 1,780 MiB | 1,779 MiB | + +The ordering matches the macOS run except initial encryption, where gcrypt is +about 6% faster on this two-core machine. The client disk total was computed +from the run's `medians.tsv` (client `.git` plus carrier cache), because that +runner revision's summary omitted the cache. + +In that run the first tiny push after the initial push took 9-32 s and about +450 MiB, because the client re-walked the full history before the push; later +pushes took about 0.2 s. After the fix in `f4526d8`, a one-round e2ee-only +rerun on the same container measured 0.15-0.16 s and 6-7 MiB for all three +tiny pushes, including the first. + +### Carrier receiver attribute sensitivity probe (2026-10-02) + +I repeated the E2EE phases once with `BENCH_ROUNDS=1 BENCH_E2EE_ONLY=1 +BENCH_CARRIER_ATTR_TREE=0 BENCH_TRACE=1 BENCH_TINY_COMMITS=5`. This leaves the +local bare carrier receiver's `attr.tree` unset while keeping +`receive.unpackLimit=0`; plain Git and gcrypt were skipped. The probe used the +same Godot and gcrypt revisions above and helper revision +`338d94ef59b8141309355007678e0e0d94a1cbf8`. Its single-run readings are +compared below with the three-round medians above, so small differences include +normal run-to-run noise. + +| E2EE phase | `attr.tree` set: 3-round median | `attr.tree` unset: one run | +|---|---:|---:| +| Initial encryption: wall time; stored bytes | 14.200 s; 920,636,512 B | 15.160 s; 920,636,512 B | +| Initial carrier push: wall time; remote bytes added | 29.460 s; 920,777,562 B | 31.080 s; 920,777,562 B | +| Fresh fetch: wall time; matching remote object bytes | 40.430 s; 920,764,176 B | 41.540 s; 920,764,396 B | +| Tiny commit: median wall time across five commits | 0.160 s | 0.150 s | +| Tiny push: median wall time; median remote bytes per push | 1.230 s; 9,460 B | 1.290 s; 9,460 B | +| Tiny update: median fetch; matching remote bytes per update | 0.890 s; 9,460 B | 0.840 s; 9,460 B | + +The receiver without `attr.tree` did not make fresh fetch materially slower: +41.54 s versus 40.43 s, a 1.11 s difference in these differently sized +samples. The traced carrier-cache fetch from the remote took 6.72 s; object +reads took 0.40 s for 920.6 MB, decryption took 15.75 s, `index-pack` took +28.34 s, and inner-ref connectivity took 3.15 s. The decryption and +`index-pack` timers overlap. Warm update fetches took 0.80--0.94 s, with remote +cache refresh around 0.08 s and incremental decrypt/import around 0.01--0.02 s. + +| Tiny operation | `attr.tree` set: earlier median | `attr.tree` unset: one run | +|---|---:|---:| +| Push 1 | 4.380 s / 9,344 B | 4.160 s / 9,346 B | +| Push 2 | 1.090 s / 9,407 B | 1.170 s / 9,405 B | +| Push 3 | 1.150 s / 9,460 B | 1.290 s / 9,460 B | +| Push 4 | 1.220 s / 9,520 B | 1.260 s / 9,518 B | +| Push 5 | 1.250 s / 9,578 B | 1.370 s / 9,577 B | +| Update fetches 1--5 | 0.730 / 0.790 / 0.910 / 0.890 / 1.050 s | 0.800 / 0.840 / 0.830 / 0.920 / 0.940 s | + +No client-side change was warranted by this result. The test client created +the uploaded carrier pack with delta search disabled, and the receiver kept +incoming packs intact with `receive.unpackLimit=0`. The 6.72 s remote fetch +shows no sign of the expensive server-side repack seen in the pre-optimization +profile; pack reuse is the likely explanation, though this run did not +instrument the server's `upload-pack` internals. A Git client cannot set a +hosted server's `upload-pack` configuration; hosted services may choose +different pack reuse and generation behavior. To reproduce the probe, the +harness accepts `BENCH_E2EE_ONLY=1 BENCH_CARRIER_ATTR_TREE=0`. + +### Historical phase profile before and after carrier optimizations + +These traced single-run figures predate the initial-encryption crypto +acceleration above. The three-round medians earlier in this section are the +current benchmark results. Pack decryption and `index-pack` timers overlap +because Git consumes the decrypted stream while it is being produced; their +durations must not be summed. + +| Operation | Before: wall and main phases | After: wall and main phases | +|---|---|---| +| Initial carrier push | 181.7 s total; cache remote fetch was small; inner pack 14.5 s; `git add e2ee` 22.7 s; carrier `git push` 143.6 s | 32.1 s total; cache fetch 0.05 s, checkout setup 0.15 s; inner pack/encryption 14.7 s; `git add` 10.3 s; commit 0.17 s; carrier push 5.9 s | +| Fresh E2EE fetch | 178.3 s total; carrier cache fetch 143.1 s; object reads 0.08 s / 920.6 MB; decrypt 15.5 s; `index-pack` 28.2 s; connectivity 3.1 s | 44.0 s total; carrier cache fetch 9.3 s; checkout setup 0.18 s; object reads 0.41 s / 920.6 MB; decrypt 15.8 s; `index-pack` 28.3 s; connectivity 3.1 s | +| First tiny push | 141.0 s total; cache remote fetch 133.9 s; checkout 1.14 s; connectivity 3.55 s; `git add` 1.79 s | 4.84 s total; cache refresh and checkout 0.28 s; connectivity 3.78 s; incremental pack 0.02 s; staging 0.01 s | +| Warm tiny push | 3.03 s total; checkout 1.16 s; refresh 0.12 s; connectivity 0.05 s; `git add` 1.08 s | 1.25 s total; checkout 0.17 s; refresh 0.11 s; connectivity 0.11 s; staging 0.01 s | +| Tiny update fetch | 1.50 s total; cache refresh about 0.14 s; checkout about 1.08 s | 0.89 s median; cache fetch 0.09 s; checkout 0.18 s; chain verification 0.13 s; connectivity 0.06 s; decrypt/import about 0.02 s | + +The most effective changes were compression-free carrier Git settings and a +committed `-delta` attribute honored by the local receiver, direct writes of +published objects into the locked cache, and a temporary checkout with an +index but no materialized historical files. Carrier push fell from 143.6 s in +the profile to 5.9 s, cache fetch from 143.1 s to 9.3 s, and the first tiny +push from 141.0 s to 4.84 s. The code keeps a disposable Git directory and +index per helper process rather than sharing a mutable index in the cache. +This avoids cross-process index races while removing the history checkout and +full-tree `git add` scan. Trace can be enabled with +`GIT_REMOTE_E2EE_TRACE=1`; the benchmark passes it and prints per-phase lines +when run with `BENCH_TRACE=1`. + +## Earlier local performance benchmarks + +These earlier measurements use the local filesystem backend or other +backend-specific scenarios. They are retained as historical comparisons and +are not a substitute for the Git-backend results above. The benchmark is opt-in and never runs in CI. It measures the release binaries against a real Git repository using an isolated filesystem backend: @@ -270,7 +631,7 @@ The results above are the clean rerun after disabling `receive.autogc` and automatic client maintenance. The tables report operation time and immediate per-push logical delta so phase boundaries remain explicit. -## Comparison with the tools in the feature table +## Earlier comparison with the tools in the feature table The tools in the README do not all encrypt at the same layer, so one combined ranking would be misleading. The following are two separate, single-run local @@ -282,15 +643,14 @@ Tool revisions were `git-remote-gcrypt` `a5ff704d071f`, `git-crypt` `8c7a90ff38fc`, `transcrypt` `1b59c8e505c0`, and `git-remote-e2ee` `f7ef09c` plus the uncommitted benchmark harnesses described here. -### Whole encrypted remote: Godot +### Earlier whole encrypted remote: Godot (gcrypt local-filesystem backend) This is the direct comparison. Each whole-remote tool received the same Godot default-branch history (85,678 commits, 14,162 HEAD files, 867 MiB reachable data), followed by the same synthetic commits. `git-remote-gcrypt` used its local-filesystem backend, which its own documentation identifies as an -efficient backend. This does **not** represent its arbitrary Git or SFTP -transport, for which its documentation warns that complete history may be -uploaded on each push. +efficient backend. It does **not** represent the Git backend measured above; +these older numbers cannot establish what gcrypt sends to a Git-hosted remote. | Metric | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | |---|---:|---:|---:| diff --git a/docs/design.md b/docs/design.md index 7c03b17..80d9731 100644 --- a/docs/design.md +++ b/docs/design.md @@ -118,6 +118,14 @@ administrative CAS is never automatically rebased. - New data is generated with `git pack-objects --stdout --revs`. - Pack output is encrypted as 1 MiB authenticated segments directly into a backend-owned stage; it is never collected into a whole-pack Rust buffer. +- Pack ciphertext hashing uses `sha2`'s runtime-selected hardware assembly + when available, with its portable fallback otherwise. The ChaCha20 backend + selects NEON automatically on supported AArch64 targets, including for + `cargo install --git` builds. The STREAM implementation is provided by + RustCrypto's `aead-stream` crate; switching to it preserves the BE32 + construction. These code-generation choices do not change the stream format + or affect Git configuration in the user's inner repository. + `GIT_REMOTE_E2EE_TRACE=1` reports the compiled ChaCha20 backend. - Existing remote tips present locally are pack exclusions. - Packs are decrypted segment by segment directly into `git index-pack` oldest-first. @@ -192,13 +200,35 @@ and membership changes are encrypted; object-size patterns can still suggest that a membership transition occurred. Inner Git metadata remains encrypted. Each helper process fetches the carrier branch into a persistent bare object -cache, then creates a disposable checkout that borrows the cache's objects. -Fetches transfer only objects missing from the cache. The cache is keyed by the -SHA-256 of a normalized remote URL; a file lock serializes updates across helper -processes, and automatic garbage collection is disabled so borrowed objects -remain available. The carrier branch is fetched again immediately before each -CAS, and the CAS remains a normal fast-forward push. A stale cache therefore -cannot authorize a write against an outdated branch tip. +cache, then creates a disposable Git directory with an alternate pointing at +the cache. It seeds a process-local index from the base commit with `git +read-tree`, without checking out or copying the historical `e2ee/` files. +Carrier object reads stream blobs through `git cat-file --batch`; publication +stages only the new chunk paths and `HEAD` with a NUL-delimited path list, so +Git does not scan the complete carrier tree. The temporary index keeps +concurrent helper processes from sharing mutable checkout state. + +The cache is keyed by the SHA-256 of a normalized remote URL; a file lock +serializes cache updates and publications across helper processes. Automatic +garbage collection is disabled so borrowed objects remain available. During a +publication, Git writes new objects and the commit into the locked cache object +database. Only after the normal fast-forward push wins does the cache's carrier +ref advance to the new commit. A lost CAS can leave unreachable ciphertext in +the cache, but it does not move the ref. The carrier branch is fetched again +immediately before each CAS, so a stale cache cannot authorize a write against +an outdated branch tip. + +Carrier Git repositories touched by the helper set `core.compression=0`, +`core.looseCompression=0`, `pack.compression=0`, `pack.window=0`, +`pack.depth=0`, and `core.bigFileThreshold=1m`. The committed root +`.gitattributes` marks `e2ee/**` as `-delta`. These settings avoid spending CPU +compressing or deltifying ciphertext, which is already encrypted and +incompressible. They are performance settings, not integrity assumptions; +normal object IDs and the signed manifest checks remain in force. A Git server +must consult that attribute tree for its own `upload-pack` delta search to +avoid server-side deltification. The benchmark's local bare receiver sets +`attr.tree=refs/heads/git-remote-e2ee`; hosted servers control this setting and +may have different server-side pack behavior. Recovery publishes a fresh cache generation instead of replacing objects that an in-flight temporary checkout may still borrow. Obsolete generations are diff --git a/scripts/benchmark-git-backend.sh b/scripts/benchmark-git-backend.sh new file mode 100755 index 0000000..678865e --- /dev/null +++ b/scripts/benchmark-git-backend.sh @@ -0,0 +1,756 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +usage() { + cat >&2 <<'EOF' +usage: benchmark-git-backend.sh GODOT_CHECKOUT GCRYPT_CHECKOUT GODOT_REVISION GCRYPT_REVISION RESULTS_DIR + +Internal measurement engine for reproduce-benchmark.sh. It requires the +explicit, revision-verified input checkouts and a results directory. +BENCH_TINY_COMMITS defaults to 5. BENCH_GCRYPT_TINY_COMMITS defaults to the +same value and may be lower for a prohibitively slow full-history run. +BENCH_ROUNDS defaults to 3. BENCH_SKIP_BUILD=1 uses existing release binaries. +BENCH_KEEP_WORK=1 retains temporary repositories and command logs under the +work directory and throwaway GPG homes under /tmp. Results contain measurement +TSV files only; no keys or command logs are written there. BENCH_TRACE=1 prints +E2EE trace lines to stderr. +BENCH_E2EE_ONLY=1 skips plain and gcrypt measurements. Set +BENCH_CARRIER_ATTR_TREE=0 to leave attr.tree unset on the local carrier +receiver, simulating servers that ignore the committed carrier attributes. +EOF +} + +if [[ $# -gt 0 && ( $1 == -h || $1 == --help ) ]]; then + usage + exit 0 +fi +if [[ $# -ne 5 ]]; then + usage + exit 2 +fi + +project_root=$(cd "$(dirname "$0")/.." && pwd -P) +source_repo=$1 +gcrypt_checkout=$2 +source_revision=$3 +gcrypt_revision=$4 +results_dir=$5 +source_repo=$(cd "$source_repo" && pwd -P) +gcrypt_checkout=$(cd "$gcrypt_checkout" && pwd -P) +results_dir=$(cd "$results_dir" && pwd -P) +[[ -d $results_dir ]] || { + echo "results directory does not exist: $results_dir" >&2 + exit 2 +} +for result_file in raw.tsv medians.tsv; do + [[ ! -e $results_dir/$result_file ]] || { + echo "results file already exists: $results_dir/$result_file" >&2 + exit 2 + } +done +git_config_count=${GIT_CONFIG_COUNT:-0} +if [[ ! $git_config_count =~ ^[0-9]+$ ]]; then + echo "GIT_CONFIG_COUNT must be a non-negative integer" >&2 + exit 2 +fi +git_config_count=$((10#$git_config_count)) +printf -v git_config_key_name 'GIT_CONFIG_KEY_%d' "$git_config_count" +printf -v git_config_value_name 'GIT_CONFIG_VALUE_%d' "$git_config_count" +printf -v "$git_config_key_name" '%s' init.defaultBranch +printf -v "$git_config_value_name" '%s' main +export "$git_config_key_name" "$git_config_value_name" +export GIT_CONFIG_COUNT=$((git_config_count + 1)) +e2ee_revision=$(git -C "$project_root" rev-parse HEAD) +tiny_commits=${BENCH_TINY_COMMITS:-5} +gcrypt_tiny_commits=${BENCH_GCRYPT_TINY_COMMITS:-$tiny_commits} +rounds=${BENCH_ROUNDS:-3} +e2ee_only=${BENCH_E2EE_ONLY:-0} +carrier_attr_tree=${BENCH_CARRIER_ATTR_TREE:-1} + +if [[ $e2ee_only != 0 && $e2ee_only != 1 ]]; then + echo "BENCH_E2EE_ONLY must be 0 or 1" >&2 + exit 2 +fi +if [[ $carrier_attr_tree != 0 && $carrier_attr_tree != 1 ]]; then + echo "BENCH_CARRIER_ATTR_TREE must be 0 or 1" >&2 + exit 2 +fi +transports=(plain gcrypt e2ee) +if [[ $e2ee_only == 1 ]]; then + transports=(e2ee) +fi + +if [[ ! $tiny_commits =~ ^[1-5]$ ]]; then + echo "BENCH_TINY_COMMITS must be from 1 to 5" >&2 + exit 2 +fi +if [[ ! $gcrypt_tiny_commits =~ ^[1-5]$ ]] || (( gcrypt_tiny_commits > tiny_commits )); then + echo "BENCH_GCRYPT_TINY_COMMITS must be from 1 to BENCH_TINY_COMMITS" >&2 + exit 2 +fi +if [[ ! $rounds =~ ^[1-3]$ ]]; then + echo "BENCH_ROUNDS must be from 1 to 3" >&2 + exit 2 +fi +[[ $(git -C "$source_repo" rev-parse HEAD) == "$source_revision" ]] || { + echo "Godot checkout must be at $source_revision" >&2 + exit 2 +} +[[ $(git -C "$source_repo" rev-parse --is-shallow-repository) == false ]] || { + echo "Godot checkout must contain full history, not a shallow clone" >&2 + exit 2 +} +[[ $(git -C "$gcrypt_checkout" rev-parse HEAD) == "$gcrypt_revision" ]] || { + echo "git-remote-gcrypt checkout must be at $gcrypt_revision" >&2 + exit 2 +} +[[ -x $gcrypt_checkout/git-remote-gcrypt ]] || { + echo "git-remote-gcrypt script is missing or not executable" >&2 + exit 1 +} +git -C "$source_repo" cat-file -e HEAD:README.md || { + echo "benchmark source must contain README.md" >&2 + exit 2 +} + +gpg_bin=$(command -v gpg || true) +[[ -n $gpg_bin && -x $gpg_bin ]] || { + echo "GnuPG executable 'gpg' was not found on PATH" >&2 + exit 1 +} +os=$(uname -s) +case $os in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform: $os" >&2; exit 2 ;; +esac +[[ -x /usr/bin/time ]] || { + echo "/usr/bin/time is required" >&2 + exit 1 +} +if [[ $os == Linux ]]; then + default_tmp=${TMPDIR:-/var/tmp} +else + default_tmp=${TMPDIR:-/tmp} +fi +work_parent=${BENCH_WORK_PARENT:-$default_tmp} +[[ -d $work_parent ]] || { + echo "BENCH_WORK_PARENT must already exist: $work_parent" >&2 + exit 2 +} +work_parent=$(cd "$work_parent" && pwd -P) +allowed_work_parent=false +case $os in + Darwin) + allowed_roots=(/private/tmp "$default_tmp") + allowed_work_description="/private/tmp or TMPDIR" + ;; + Linux) + allowed_roots=(/var/tmp /tmp "$default_tmp") + allowed_work_description="/var/tmp, /tmp, or TMPDIR" + ;; +esac +for allowed_root in "${allowed_roots[@]}"; do + [[ -n $allowed_root ]] || continue + [[ -d $allowed_root ]] || continue + allowed_root=$(cd "$allowed_root" && pwd -P) + if [[ $work_parent == "$allowed_root" || $work_parent == "$allowed_root"/* ]]; then + allowed_work_parent=true + break + fi +done +if [[ $allowed_work_parent != true ]]; then + echo "benchmark work parent must resolve under $allowed_work_description: $work_parent" >&2 + exit 2 +fi +case $os in + Darwin) + work_filesystem_device=$(df -P "$work_parent" | awk 'NR == 2 { print $1 }') + work_filesystem=$(mount | awk -v device="$work_filesystem_device" '$1 == device { + if (match($0, /\([^,]+/)) { + print substr($0, RSTART + 1, RLENGTH - 1) + exit + } + }') + ;; + Linux) + work_filesystem=$(stat -f -c '%T' "$work_parent") + ;; +esac +case $work_filesystem in + tmpfs|ramfs) + echo "benchmark work directory is on $work_filesystem: $work_parent" >&2 + echo "Choose disk-backed storage by setting TMPDIR, or set BENCH_WORK_PARENT under /var/tmp or TMPDIR." >&2 + exit 2 + ;; +esac +source_git_kib=$(du -sk "$source_repo/.git" | awk '{print $1}') +available_kib=$(df -Pk "$work_parent" | + awk 'NR == 2 {print $4}') +if (( available_kib < source_git_kib * 12 )); then + echo "need at least 12x the Godot .git size free under the benchmark temporary directory" >&2 + exit 1 +fi +gpg_home_root=$(cd /tmp && pwd -P) +gpg_agent_socket_path="$gpg_home_root/git-remote-e2ee-gnupg.XXXXXX/S.gpg-agent" +if (( ${#gpg_agent_socket_path} >= 90 )); then + echo "the canonical /tmp path is too long for a GnuPG agent socket: $gpg_home_root" >&2 + exit 2 +fi + +if [[ ${BENCH_SKIP_BUILD:-0} != 1 ]]; then + cargo build --release --bins --manifest-path "$project_root/Cargo.toml" +fi +git_e2ee="$project_root/target/release/git-e2ee" +[[ -x $git_e2ee && -x $project_root/target/release/git-remote-e2ee ]] || { + echo "release helper binaries are missing under target/release" >&2 + exit 1 +} + +export PATH="$gcrypt_checkout:$project_root/target/release:$PATH" +if [[ ${BENCH_TRACE:-0} == 1 ]]; then + export GIT_REMOTE_E2EE_TRACE=1 +fi +work=$(mktemp -d "$work_parent/git-remote-e2ee-bench.XXXXXX") +work=$(cd "$work" && pwd -P) +chmod 700 "$work" +gpgconf_bin=$(command -v gpgconf || true) +active_gpg_home= +cleanup_gpg_home() { + local path=$1 + case "$path" in + "$gpg_home_root"/git-remote-e2ee-gnupg.*) + if [[ $(cd "$(dirname "$path")" && pwd -P) == "$gpg_home_root" && -d $path && ! -L $path ]]; then + rm -rf "$path" + else + echo "refusing to remove unexpected GPG home: $path" >&2 + fi + ;; + *) echo "refusing to remove unexpected GPG home: $path" >&2 ;; + esac +} +cleanup() { + if [[ -n $active_gpg_home && -n $gpgconf_bin ]]; then + "$gpgconf_bin" --homedir "$active_gpg_home" --kill all >/dev/null 2>&1 || true + fi + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + if [[ -n $active_gpg_home ]]; then + echo "throwaway GPG home retained at $active_gpg_home" >&2 + fi + return + fi + if [[ -n $active_gpg_home ]]; then + cleanup_gpg_home "$active_gpg_home" + active_gpg_home= + fi + case "$work" in + "$work_parent"/git-remote-e2ee-bench.*) + if [[ $(cd "$(dirname "$work")" && pwd -P) == "$work_parent" && ! -L $work ]]; then + rm -rf "$work" + else + echo "refusing to remove unexpected benchmark path: $work" >&2 + fi + ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +raw_tsv="$work/measurements.tsv" +printf 'round\ttransport\tphase\titeration\twall_seconds\tmax_rss_bytes\tremote_or_stored_delta_bytes\tremote_or_stored_total_logical_bytes\tremote_or_stored_total_allocated_bytes\tclient_git_logical_bytes\tclient_git_allocated_bytes\tlocal_state_logical_bytes\tlocal_state_allocated_bytes\tclient_object_store_growth\n' >"$raw_tsv" + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {printf "%.0f\n", sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {printf "%.0f\n", sum + 0}' + fi +} + +tree_allocated_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + else + du -sk "$path" | awk '{printf "%.0f\n", $1 * 1024}' + fi +} + +git_object_db_bytes() { + local repo=$1 git_dir + if [[ ! -d $repo ]]; then + echo 0 + return + fi + git_dir=$(git -C "$repo" rev-parse --absolute-git-dir) + tree_logical_bytes "$git_dir/objects" +} + +git_bare_object_db_bytes() { + local repo=$1 + tree_logical_bytes "$repo/objects" +} + +carrier_cache_object_db_bytes() { + local root=$1 config repo value total=0 + [[ -d $root ]] || { echo 0; return; } + while IFS= read -r config; do + repo=${config%/config} + if [[ $(git -C "$repo" rev-parse --is-bare-repository 2>/dev/null || true) == true ]]; then + value=$(git_object_db_bytes "$repo") + total=$((total + value)) + fi + done < <(find "$root" -type f -name config -print) + echo "$total" +} + +measure() { + local label=$1 timing="$work/$1.time" output="$work/$1.stdout" error="$work/$1.stderr" + shift + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$output" 2>"$error"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$error" >&2 + sed -n '1,100p' "$output" >&2 + return 1 + fi + MEASURED_WALL=$(awk '/ real / {print $1; exit}' "$timing") + MEASURED_RSS_BYTES=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$output" 2>"$error"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$error" >&2 + sed -n '1,100p' "$output" >&2 + return 1 + fi + MEASURED_WALL=$(awk -F': ' '/Elapsed \(wall clock\)/ { + n = split($2, a, ":") + if (n == 3) printf "%.3f\n", a[1] * 3600 + a[2] * 60 + a[3] + else printf "%.3f\n", a[1] * 60 + a[2] + }' "$timing") + MEASURED_RSS_BYTES=$(awk -F': ' '/Maximum resident set size \(kbytes\)/ { + printf "%.0f\n", $2 * 1024 + exit + }' "$timing") + fi + if [[ -z $MEASURED_RSS_BYTES ]]; then + echo "could not read max RSS from $timing" >&2 + return 1 + fi + if [[ ${BENCH_TRACE:-0} == 1 && $label == *e2ee* ]]; then + awk -v phase="$label" '/^git-remote-e2ee trace / { + printf "benchmark_trace phase=%s %s\n", phase, $0 + }' "$error" >&2 + fi +} + +record() { + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$@" >>"$raw_tsv" +} + +record_sizes() { + local round=$1 transport=$2 phase=$3 iteration=$4 wall=$5 max_rss=$6 + local remote_delta=$7 remote_path=$8 client_path=$9 state_path=${10} growth=${11} + local remote_logical remote_allocated client_logical=0 client_allocated=0 + local state_logical=0 state_allocated=0 + remote_logical=$(tree_logical_bytes "$remote_path") + remote_allocated=$(tree_allocated_bytes "$remote_path") + if [[ -n $client_path ]]; then + client_logical=$(tree_logical_bytes "$client_path") + client_allocated=$(tree_allocated_bytes "$client_path") + fi + if [[ -n $state_path ]]; then + state_logical=$(tree_logical_bytes "$state_path") + state_allocated=$(tree_allocated_bytes "$state_path") + fi + record "$round" "$transport" "$phase" "$iteration" "$wall" "$max_rss" \ + "$remote_delta" "$remote_logical" "$remote_allocated" \ + "$client_logical" "$client_allocated" "$state_logical" \ + "$state_allocated" "$growth" +} + +run_round() { + local round=$1 round_dir="$work/$1" + local branch=main bare_local plain_git gcrypt_dir gcrypt_git carrier_git + local gcrypt_url e2ee_url key_directory key_carrier fingerprint gpg_home bare_repo + local source_work plain_fresh gcrypt_fresh e2ee_fresh + local cache_push cache_returning wall rss before after secondary i transport + local remote_path state_path + + mkdir -m 700 "$round_dir" + source_work="$round_dir/source" + bare_local="$round_dir/plain-local.git" + plain_git="$round_dir/plain-git.git" + gcrypt_dir="$round_dir/gcrypt-directory" + gcrypt_git="$round_dir/gcrypt-git.git" + carrier_git="$round_dir/e2ee-carrier.git" + plain_fresh="$round_dir/plain-returning" + gcrypt_fresh="$round_dir/gcrypt-returning" + e2ee_fresh="$round_dir/e2ee-returning" + cache_push="$round_dir/cache/push" + cache_returning="$round_dir/cache/returning" + gpg_home=$(mktemp -d "$gpg_home_root/git-remote-e2ee-gnupg.XXXXXX") + active_gpg_home=$gpg_home + + export GNUPGHOME="$gpg_home" + if ! "$gpg_bin" --batch --pinentry-mode loopback --passphrase '' \ + --quick-generate-key 'Benchmark ' ed25519 sign 0 \ + >"$gpg_home/keygen.stdout" 2>"$gpg_home/keygen.stderr"; then + echo "throwaway GPG key generation failed" >&2 + sed -n '1,100p' "$gpg_home/keygen.stderr" >&2 + sed -n '1,100p' "$gpg_home/keygen.stdout" >&2 + return 1 + fi + fingerprint=$("$gpg_bin" --batch --with-colons --list-secret-keys | + awk -F: '$1 == "fpr" {print $10; exit}') + [[ -n $fingerprint ]] + if ! "$gpg_bin" --batch --pinentry-mode loopback --passphrase '' \ + --quick-add-key "$fingerprint" cv25519 encr 0 \ + >"$gpg_home/subkey.stdout" 2>"$gpg_home/subkey.stderr"; then + echo "throwaway GPG encryption subkey generation failed" >&2 + sed -n '1,100p' "$gpg_home/subkey.stderr" >&2 + sed -n '1,100p' "$gpg_home/subkey.stdout" >&2 + return 1 + fi + + git clone --quiet --no-local --no-checkout --single-branch --no-tags \ + "file://$source_repo" "$source_work" + git -C "$source_work" checkout --quiet -B "$branch" "$source_revision" + git -C "$source_work" checkout --quiet -- README.md + git -C "$source_work" config gc.auto 0 + git -C "$source_work" config maintenance.auto false + git -C "$source_work" config gpg.program "$gpg_bin" + git -C "$source_work" config user.name Benchmark + git -C "$source_work" config user.email benchmark@example.invalid + + for bare_repo in "$bare_local" "$plain_git" "$gcrypt_git" "$carrier_git"; do + git init --bare --quiet --initial-branch=main "$bare_repo" + git --git-dir="$bare_repo" config gc.auto 0 + git --git-dir="$bare_repo" config maintenance.auto false + git --git-dir="$bare_repo" config receive.autogc false + git --git-dir="$bare_repo" config receive.unpackLimit 0 + done + git --git-dir="$gcrypt_git" symbolic-ref HEAD refs/heads/gcrypt-carrier + if [[ $carrier_attr_tree == 1 ]]; then + git --git-dir="$carrier_git" config attr.tree refs/heads/git-remote-e2ee + else + git --git-dir="$carrier_git" config --unset-all attr.tree 2>/dev/null || true + fi + + key_directory="$round_dir/directory.key.json" + key_carrier="$round_dir/carrier.key.json" + "$git_e2ee" keygen --output "$key_directory" >/dev/null 2>&1 + "$git_e2ee" keygen --output "$key_carrier" >/dev/null 2>&1 + gcrypt_url="gcrypt::file://$gcrypt_git#gcrypt-carrier" + e2ee_url="e2ee::git+file://$carrier_git" + + git -C "$source_work" remote add plain-local "file://$bare_local" + git -C "$source_work" remote add gcrypt-directory "gcrypt::$gcrypt_dir" + git -C "$source_work" remote add plain-git "file://$plain_git" + git -C "$source_work" remote add gcrypt-git "$gcrypt_url" + git -C "$source_work" remote add e2ee-directory "e2ee::$round_dir/e2ee-directory" + git -C "$source_work" remote add e2ee-carrier "$e2ee_url" + + for remote_name in gcrypt-directory gcrypt-git; do + git -C "$source_work" config "remote.$remote_name.gcrypt-participants" "$fingerprint" + git -C "$source_work" config "remote.$remote_name.gcrypt-signingkey" "$fingerprint" + git -C "$source_work" config "remote.$remote_name.gcrypt-require-explicit-force-push" true + done + git -C "$source_work" config remote.e2ee-directory.e2ee-key "$key_directory" + git -C "$source_work" config remote.e2ee-carrier.e2ee-key "$key_carrier" + "$git_e2ee" init --storage "$round_dir/e2ee-directory" --key "$key_directory" >/dev/null + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_push" \ + "$git_e2ee" carrier-init --remote "$carrier_git" --key "$key_carrier" >/dev/null + + echo "round $round: initial local encryption and initial Git-backend pushes" >&2 + if [[ $e2ee_only != 1 ]]; then + measure "$round-initial-local-plain" \ + git -C "$source_work" push --quiet plain-local "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$bare_local") + record_sizes "$round" plain initial_encryption 0 "$wall" "$rss" \ + "$after" "$bare_local" "$source_work/.git" "" 0 + + measure "$round-initial-local-gcrypt" \ + git -C "$source_work" push --quiet --force gcrypt-directory "$branch:master" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(tree_logical_bytes "$gcrypt_dir") + record_sizes "$round" gcrypt initial_encryption 0 "$wall" "$rss" \ + "$after" "$gcrypt_dir" "$source_work/.git" "$source_work/.git/remote-gcrypt" 0 + fi + + measure "$round-initial-local-e2ee" \ + git -C "$source_work" push --quiet e2ee-directory "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(tree_logical_bytes "$round_dir/e2ee-directory") + record_sizes "$round" e2ee initial_encryption 0 "$wall" "$rss" \ + "$after" "$round_dir/e2ee-directory" "$source_work/.git" "" 0 + + if [[ $e2ee_only != 1 ]]; then + before=$(git_bare_object_db_bytes "$plain_git") + measure "$round-initial-plain-push" \ + git -C "$source_work" push --quiet plain-git "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$plain_git") + record_sizes "$round" plain initial_push 0 "$wall" "$rss" \ + "$((after - before))" "$plain_git" "$source_work/.git" "" 0 + + before=$(git_bare_object_db_bytes "$gcrypt_git") + measure "$round-initial-gcrypt-git-push" \ + git -C "$source_work" push --quiet --force gcrypt-git "$branch:master" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$gcrypt_git") + record_sizes "$round" gcrypt initial_push 0 "$wall" "$rss" \ + "$((after - before))" "$gcrypt_git" "$source_work/.git" \ + "$source_work/.git/remote-gcrypt" 0 + fi + + before=$(git_bare_object_db_bytes "$carrier_git") + measure "$round-initial-e2ee-carrier-push" \ + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_push" \ + git -C "$source_work" push --quiet e2ee-carrier "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$carrier_git") + record_sizes "$round" e2ee initial_push 0 "$wall" "$rss" \ + "$((after - before))" "$carrier_git" "$source_work/.git" "$cache_push" 0 + + echo "round $round: fresh no-checkout clones from Git backends" >&2 + if [[ $e2ee_only != 1 ]]; then + measure "$round-fresh-plain-clone" git \ + -c gc.auto=0 -c maintenance.auto=false \ + clone --quiet --no-local --no-tags --no-checkout --branch "$branch" \ + "file://$plain_git" "$plain_fresh" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_object_db_bytes "$plain_fresh") + record_sizes "$round" plain fresh_fetch 0 "$wall" "$rss" \ + "$(git_bare_object_db_bytes "$plain_git")" "$plain_git" \ + "$plain_fresh/.git" "" "$after" + + measure "$round-fresh-gcrypt-clone" git \ + -c gc.auto=0 -c maintenance.auto=false -c "gpg.program=$gpg_bin" \ + -c "remote.origin.gcrypt-participants=$fingerprint" \ + -c "remote.origin.gcrypt-signingkey=$fingerprint" \ + clone --quiet --no-local --no-tags --no-checkout --branch master \ + "$gcrypt_url" "$gcrypt_fresh" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + git -C "$gcrypt_fresh" config gc.auto 0 + git -C "$gcrypt_fresh" config maintenance.auto false + after=$(git_object_db_bytes "$gcrypt_fresh") + record_sizes "$round" gcrypt fresh_fetch 0 "$wall" "$rss" \ + "$(git_bare_object_db_bytes "$gcrypt_git")" "$gcrypt_git" \ + "$gcrypt_fresh/.git" "$gcrypt_fresh/.git/remote-gcrypt" "$after" + fi + + before=$(carrier_cache_object_db_bytes "$cache_returning") + measure "$round-fresh-e2ee-clone" env \ + "GIT_REMOTE_E2EE_CACHE_DIR=$cache_returning" \ + git -c gc.auto=0 -c maintenance.auto=false -c "e2ee.key=$key_carrier" \ + clone --quiet --no-local --no-tags --no-checkout --branch "$branch" \ + "$e2ee_url" "$e2ee_fresh" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + if [[ $e2ee_only != 1 ]]; then + git -C "$plain_fresh" config gc.auto 0 + git -C "$plain_fresh" config maintenance.auto false + fi + git -C "$e2ee_fresh" config gc.auto 0 + git -C "$e2ee_fresh" config maintenance.auto false + after=$(carrier_cache_object_db_bytes "$cache_returning") + secondary=$(git_object_db_bytes "$e2ee_fresh") + record_sizes "$round" e2ee fresh_fetch 0 "$wall" "$rss" \ + "$((after - before))" "$carrier_git" "$e2ee_fresh/.git" \ + "$cache_returning" "$secondary" + + for ((i = 1; i <= tiny_commits; i++)); do + local plain_added=0 gcrypt_added=0 e2ee_added=0 + measure "$round-tiny-commit-$i" bash -c ' + set -euo pipefail + repo=$1 + iteration=$2 + readme="$repo/README.md" + temporary="$repo/README.md.benchmark" + { printf "Godot Git backend benchmark update %s\\n" "$iteration"; tail -n +2 "$readme"; } >"$temporary" + mv "$temporary" "$readme" + git -C "$repo" add -- README.md + git -C "$repo" commit --quiet -m "benchmark: tiny update $iteration" + ' _ "$source_work" "$i" + for transport in "${transports[@]}"; do + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + case $transport in + plain) remote_path=$plain_git; state_path= ;; + gcrypt) remote_path=$gcrypt_git; state_path="$source_work/.git/remote-gcrypt" ;; + e2ee) remote_path=$carrier_git; state_path=$cache_push ;; + esac + record_sizes "$round" "$transport" tiny_commit "$i" "$wall" "$rss" \ + 0 "$remote_path" "$source_work/.git" "$state_path" 0 + done + + if [[ $e2ee_only != 1 ]]; then + before=$(git_bare_object_db_bytes "$plain_git") + measure "$round-tiny-plain-push-$i" \ + git -C "$source_work" push --quiet plain-git "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$plain_git") + plain_added=$((after - before)) + record_sizes "$round" plain tiny_push "$i" "$wall" "$rss" \ + "$plain_added" "$plain_git" "$source_work/.git" "" 0 + fi + + if [[ $e2ee_only != 1 ]] && (( i <= gcrypt_tiny_commits )); then + before=$(git_bare_object_db_bytes "$gcrypt_git") + measure "$round-tiny-gcrypt-push-$i" \ + git -C "$source_work" push --quiet --force gcrypt-git "$branch:master" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$gcrypt_git") + gcrypt_added=$((after - before)) + record_sizes "$round" gcrypt tiny_push "$i" "$wall" "$rss" \ + "$gcrypt_added" "$gcrypt_git" "$source_work/.git" \ + "$source_work/.git/remote-gcrypt" 0 + fi + + before=$(git_bare_object_db_bytes "$carrier_git") + measure "$round-tiny-e2ee-push-$i" \ + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_push" \ + git -C "$source_work" push --quiet e2ee-carrier "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$carrier_git") + e2ee_added=$((after - before)) + record_sizes "$round" e2ee tiny_push "$i" "$wall" "$rss" \ + "$e2ee_added" "$carrier_git" "$source_work/.git" "$cache_push" 0 + + if [[ $e2ee_only != 1 ]]; then + before=$(git_object_db_bytes "$plain_fresh") + measure "$round-tiny-plain-fetch-$i" \ + git -C "$plain_fresh" fetch --quiet origin + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_object_db_bytes "$plain_fresh") + record_sizes "$round" plain tiny_update "$i" "$wall" "$rss" \ + "$plain_added" "$plain_git" "$plain_fresh/.git" "" \ + "$((after - before))" + fi + + if [[ $e2ee_only != 1 ]] && (( i <= gcrypt_tiny_commits )); then + before=$(git_object_db_bytes "$gcrypt_fresh") + measure "$round-tiny-gcrypt-fetch-$i" \ + git -C "$gcrypt_fresh" fetch --quiet origin + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_object_db_bytes "$gcrypt_fresh") + record_sizes "$round" gcrypt tiny_update "$i" "$wall" "$rss" \ + "$gcrypt_added" "$gcrypt_git" "$gcrypt_fresh/.git" \ + "$gcrypt_fresh/.git/remote-gcrypt" "$((after - before))" + fi + + before=$(carrier_cache_object_db_bytes "$cache_returning") + secondary_before=$(git_object_db_bytes "$e2ee_fresh") + measure "$round-tiny-e2ee-fetch-$i" \ + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_returning" \ + git -C "$e2ee_fresh" fetch --quiet origin + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(carrier_cache_object_db_bytes "$cache_returning") + secondary=$(git_object_db_bytes "$e2ee_fresh") + record_sizes "$round" e2ee tiny_update "$i" "$wall" "$rss" \ + "$e2ee_added" "$carrier_git" "$e2ee_fresh/.git" \ + "$cache_returning" "$((secondary - secondary_before))" + done + + if [[ $e2ee_only != 1 ]]; then + [[ $(git -C "$plain_fresh" rev-parse "refs/remotes/origin/$branch") == \ + $(git -C "$source_work" rev-parse "$branch") ]] + [[ $(git -C "$gcrypt_fresh" rev-parse refs/remotes/origin/master) == \ + $(git -C "$source_work" rev-parse "$branch~$((tiny_commits - gcrypt_tiny_commits))") ]] + fi + [[ $(git -C "$e2ee_fresh" rev-parse "refs/remotes/origin/$branch") == \ + $(git -C "$source_work" rev-parse "$branch") ]] + + if [[ -n $gpgconf_bin ]]; then + "$gpgconf_bin" --homedir "$gpg_home" --kill all >/dev/null 2>&1 || true + fi + unset GNUPGHOME + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "throwaway GPG home retained at $gpg_home" >&2 + else + cleanup_gpg_home "$gpg_home" + fi + active_gpg_home= + if [[ ${BENCH_KEEP_WORK:-0} != 1 ]]; then + case "$round_dir" in + "$work"/[1-3]) + if [[ -d $round_dir && ! -L $round_dir ]]; then + rm -rf "$round_dir" + fi + ;; + *) echo "refusing to remove unexpected round path: $round_dir" >&2 ;; + esac + fi +} + +median_value() { + local transport=$1 phase=$2 field=$3 iteration=${4:-} + awk -F '\t' -v t="$transport" -v p="$phase" -v i="$iteration" -v f="$field" ' + NR > 1 && $2 == t && $3 == p && (i == "" || $4 == i) { print $f } + ' "$raw_tsv" | sort -n | awk ' + { values[NR] = $1 } + END { + if (NR == 0) { print "-"; exit } + if (NR % 2 == 1) printf "%.3f", values[(NR + 1) / 2] + else printf "%.3f", (values[NR / 2] + values[NR / 2 + 1]) / 2 + } + ' +} + +for ((round = 1; round <= rounds; round++)); do + run_round "$round" +done + +medians_tsv="$work/medians.tsv" +printf 'transport\tphase\titeration\tmedian_wall_seconds\tmedian_max_rss_bytes\tmedian_remote_delta_bytes\tmedian_remote_total_logical_bytes\tmedian_remote_total_allocated_bytes\tmedian_client_git_logical_bytes\tmedian_client_git_allocated_bytes\tmedian_local_state_logical_bytes\tmedian_local_state_allocated_bytes\tmedian_client_object_store_growth\n' >"$medians_tsv" +for phase in initial_encryption initial_push fresh_fetch tiny_commit tiny_push tiny_update; do + for transport in "${transports[@]}"; do + printf '%s\t%s\tall' "$transport" "$phase" >>"$medians_tsv" + for field in 5 6 7 8 9 10 11 12 13 14; do + printf '\t%s' "$(median_value "$transport" "$phase" "$field")" >>"$medians_tsv" + done + printf '\n' >>"$medians_tsv" + done +done + +for phase in tiny_push tiny_update; do + for ((i = 1; i <= tiny_commits; i++)); do + for transport in "${transports[@]}"; do + printf '%s\t%s\t%s' "$transport" "$phase" "$i" >>"$medians_tsv" + for field in 5 6 7 8 9 10 11 12 13 14; do + printf '\t%s' "$(median_value "$transport" "$phase" "$field" "$i")" >>"$medians_tsv" + done + printf '\n' >>"$medians_tsv" + done + done +done + +cp "$raw_tsv" "$results_dir/raw.tsv" +cp "$medians_tsv" "$results_dir/medians.tsv" +printf 'Wrote %s and %s\n' "$results_dir/raw.tsv" "$results_dir/medians.tsv" +if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "raw command logs, repositories, and GPG homes retained under $work" >&2 +else + echo "raw command logs, repositories, and GPG homes were removed from $work" >&2 +fi diff --git a/scripts/reproduce-benchmark.sh b/scripts/reproduce-benchmark.sh new file mode 100755 index 0000000..95b7e6c --- /dev/null +++ b/scripts/reproduce-benchmark.sh @@ -0,0 +1,579 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +export LC_ALL=C + +usage() { + cat <<'EOF' +usage: scripts/reproduce-benchmark.sh + +Fetches the pinned Godot and git-remote-gcrypt histories, runs the full +three-round benchmark, and writes raw.tsv, medians.tsv, environment.json, and +summary.md under ./bench-results/-/. + +Set BENCH_CACHE_DIR to override the input cache. Set BENCH_RESULTS_DIR to +choose an exact results directory. Set BENCH_WORK_PARENT to choose a temporary +work directory under an allowed temporary root. Linux defaults to /var/tmp; +TMPDIR overrides the default when set. tmpfs and ramfs work directories are +rejected. BENCH_ROUNDS defaults to 3, BENCH_TINY_COMMITS defaults to 5, and +BENCH_GCRYPT_TINY_COMMITS defaults to BENCH_TINY_COMMITS. +EOF +} + +if [[ $# -gt 0 ]]; then + if [[ $# == 1 && ( $1 == -h || $1 == --help ) ]]; then + usage + exit 0 + fi + usage >&2 + exit 2 +fi + +project_root=$(cd "$(dirname "$0")/.." && pwd -P) +godot_url=https://github.com/godotengine/godot.git +gcrypt_url=https://github.com/spwhitton/git-remote-gcrypt.git +godot_revision=00932449c9f372b30301d8b5fdc1be70ec12b5c0 +gcrypt_revision=a5ff704d071f14b95b6b1fa0caa8cdbf0c6cdadb + +rounds=${BENCH_ROUNDS:-3} +tiny_commits=${BENCH_TINY_COMMITS:-5} +gcrypt_tiny_commits=${BENCH_GCRYPT_TINY_COMMITS:-$tiny_commits} +e2ee_only=${BENCH_E2EE_ONLY:-0} +carrier_attr_tree=${BENCH_CARRIER_ATTR_TREE:-1} +trace=${BENCH_TRACE:-0} +skip_build=${BENCH_SKIP_BUILD:-0} +keep_work=${BENCH_KEEP_WORK:-0} + +if [[ ! $rounds =~ ^[1-3]$ ]]; then + echo "BENCH_ROUNDS must be from 1 to 3" >&2 + exit 2 +fi +if [[ ! $tiny_commits =~ ^[1-5]$ ]]; then + echo "BENCH_TINY_COMMITS must be from 1 to 5" >&2 + exit 2 +fi +if [[ ! $gcrypt_tiny_commits =~ ^[1-5]$ ]] || (( gcrypt_tiny_commits > tiny_commits )); then + echo "BENCH_GCRYPT_TINY_COMMITS must be from 1 to BENCH_TINY_COMMITS" >&2 + exit 2 +fi +validate_switch() { + local name=$1 value=$2 + if [[ $value != 0 && $value != 1 ]]; then + echo "$name must be 0 or 1" >&2 + exit 2 + fi +} +validate_switch BENCH_E2EE_ONLY "$e2ee_only" +validate_switch BENCH_CARRIER_ATTR_TREE "$carrier_attr_tree" +validate_switch BENCH_TRACE "$trace" +validate_switch BENCH_SKIP_BUILD "$skip_build" +validate_switch BENCH_KEEP_WORK "$keep_work" + +os=$(uname -s) +case $os in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported operating system: $os (supported: macOS and Linux)" >&2; exit 2 ;; +esac + +missing=() +for tool in git gpg cargo rustc du df awk find sort stat mktemp hostname date sed tr tail wc chmod cp mv mkdir rmdir rm cat dirname; do + if ! command -v "$tool" >/dev/null 2>&1; then + printf "missing required command on PATH: %s\n" "$tool" >&2 + missing+=("$tool") + fi +done +if [[ $os == Darwin ]] && ! command -v mount >/dev/null 2>&1; then + echo "missing required command on PATH: mount" >&2 + missing+=(mount) +fi +if [[ ! -x /usr/bin/time ]]; then + echo "missing required command: /usr/bin/time" >&2 + missing+=(time) +fi +if (( ${#missing[@]} > 0 )); then + echo "Install the missing tools, then rerun the benchmark." >&2 + echo "Debian/Ubuntu: sudo apt-get install git gnupg cargo rustc time coreutils findutils gawk sed" >&2 + echo "Homebrew: brew install git gnupg rust gawk" >&2 + exit 1 +fi + +case $time_style in + bsd) + if ! /usr/bin/time -l -o /dev/null true >/dev/null 2>&1; then + echo "/usr/bin/time must support BSD -l output on macOS" >&2 + echo "Debian/Ubuntu: sudo apt-get install time" >&2 + echo "Homebrew: macOS normally supplies BSD /usr/bin/time; restore the system tool." >&2 + exit 1 + fi + ;; + gnu) + if ! /usr/bin/time -v -o /dev/null true >/dev/null 2>&1; then + echo "/usr/bin/time must support GNU -v output on Linux" >&2 + echo "Debian/Ubuntu: sudo apt-get install time" >&2 + echo "Homebrew: brew install gnu-time" >&2 + exit 1 + fi + ;; +esac + +gpg_bin=$(command -v gpg) +git_version=$(git --version) +gpg_version=$(gpg --version | sed -n '1p') +rustc_version=$(rustc --version) +cargo_version=$(cargo --version) +e2ee_revision=$(git -C "$project_root" rev-parse HEAD) +if [[ -n $(git -C "$project_root" status --porcelain --untracked-files=normal) ]]; then + e2ee_dirty=true +else + e2ee_dirty=false +fi + +if [[ $os == Linux ]]; then + default_tmp=${TMPDIR:-/var/tmp} +else + default_tmp=${TMPDIR:-/tmp} +fi +work_parent=${BENCH_WORK_PARENT:-$default_tmp} +[[ -d $work_parent ]] || { + echo "BENCH_WORK_PARENT must already exist: $work_parent" >&2 + exit 2 +} +work_parent=$(cd "$work_parent" && pwd -P) +allowed_work_parent=false +case $os in + Darwin) + allowed_roots=(/private/tmp "$default_tmp") + allowed_work_description="/private/tmp or TMPDIR" + ;; + Linux) + allowed_roots=(/var/tmp /tmp "$default_tmp") + allowed_work_description="/var/tmp, /tmp, or TMPDIR" + ;; +esac +for allowed_root in "${allowed_roots[@]}"; do + [[ -d $allowed_root ]] || continue + allowed_root=$(cd "$allowed_root" && pwd -P) + if [[ $work_parent == "$allowed_root" || $work_parent == "$allowed_root"/* ]]; then + allowed_work_parent=true + break + fi +done +if [[ $allowed_work_parent != true ]]; then + echo "BENCH_WORK_PARENT must resolve under $allowed_work_description: $work_parent" >&2 + exit 2 +fi + +case $os in + Darwin) + work_filesystem_device=$(df -P "$work_parent" | awk 'NR == 2 { print $1 }') + filesystem=$(mount | awk -v device="$work_filesystem_device" '$1 == device { + if (match($0, /\([^,]+/)) { + print substr($0, RSTART + 1, RLENGTH - 1) + exit + } + }') + ;; + Linux) + filesystem=$(stat -f -c '%T' "$work_parent") + ;; +esac +if [[ -z $filesystem ]]; then + echo "could not identify the filesystem for $work_parent" >&2 + exit 1 +fi +case $filesystem in + tmpfs|ramfs) + echo "benchmark work directory is on $filesystem: $work_parent" >&2 + echo "Choose disk-backed storage by setting TMPDIR, or set BENCH_WORK_PARENT under /var/tmp or TMPDIR." >&2 + exit 2 + ;; +esac + +cache_default=${XDG_CACHE_HOME:-${HOME:?HOME must be set}/.cache}/git-remote-e2ee-bench +cache_dir=${BENCH_CACHE_DIR:-$cache_default} +mkdir -p "$cache_dir" +cache_dir=$(cd "$cache_dir" && pwd -P) +godot_cache="$cache_dir/godot-$godot_revision" +gcrypt_cache="$cache_dir/git-remote-gcrypt-$gcrypt_revision" +stage_dir= +cache_lock= +cache_lock_owned=false +results_dir= +results_created=false +run_complete=false + +cleanup() { + if [[ -n $stage_dir ]]; then + case $stage_dir in + "$cache_dir"/.godot-fetch.*|"$cache_dir"/.gcrypt-fetch.*) + [[ ! -L $stage_dir ]] && rm -rf "$stage_dir" + ;; + *) echo "refusing to remove unexpected cache staging path: $stage_dir" >&2 ;; + esac + fi + if [[ $cache_lock_owned == true && -n $cache_lock && -d $cache_lock && ! -L $cache_lock ]]; then + rmdir "$cache_lock" 2>/dev/null || true + fi + if [[ $results_created == true && $run_complete != true && -d $results_dir ]]; then + rmdir "$results_dir" 2>/dev/null || true + fi +} +trap cleanup EXIT + +validate_single_branch_cache() { + local path=$1 revision=$2 name=$3 default_ref branch_count + [[ -d $path && ! -L $path ]] || { + echo "$name cache is not a directory: $path" >&2 + return 1 + } + [[ $(git -C "$path" rev-parse HEAD) == "$revision" ]] || { + echo "$name cache has the wrong revision at $path; refusing to modify it" >&2 + return 1 + } + [[ $(git -C "$path" rev-parse --is-shallow-repository) == false ]] || { + echo "$name cache is shallow at $path; choose a new BENCH_CACHE_DIR" >&2 + return 1 + } + default_ref=$(git -C "$path" symbolic-ref --quiet refs/remotes/origin/HEAD || true) + [[ $default_ref == refs/remotes/origin/* ]] || { + echo "$name cache does not identify its default branch: $path" >&2 + return 1 + } + branch_count=$(git -C "$path" for-each-ref --format='%(refname)' refs/remotes/origin | + awk '$0 != "refs/remotes/origin/HEAD" { count++ } END { print count + 0 }') + [[ $branch_count == 1 ]] || { + echo "$name cache must contain only its default branch; choose a new BENCH_CACHE_DIR" >&2 + return 1 + } + [[ -z $(git -C "$path" tag --list) ]] || { + echo "$name cache contains tags; choose a new BENCH_CACHE_DIR" >&2 + return 1 + } + git -C "$path" merge-base --is-ancestor "$revision" "$default_ref" || { + echo "$name pinned revision is not reachable from the cached default branch" >&2 + return 1 + } +} + +publish_cache() { + local stage=$1 destination=$2 name=$3 + cache_lock="$destination.publish-lock" + if ! mkdir "$cache_lock" 2>/dev/null; then + echo "$name cache publication is already in progress or has a stale lock: $cache_lock" >&2 + echo "If no benchmark is running, remove that lock directory and rerun." >&2 + return 1 + fi + cache_lock_owned=true + if [[ -e $destination || -L $destination ]]; then + validate_single_branch_cache "$destination" "$4" "$name" + rm -rf "$stage" + stage_dir= + else + mv "$stage/repo" "$destination" + rmdir "$stage" + stage_dir= + fi + rmdir "$cache_lock" + cache_lock= + cache_lock_owned=false +} + +if [[ -e $godot_cache || -L $godot_cache ]]; then + validate_single_branch_cache "$godot_cache" "$godot_revision" Godot + echo "Reusing Godot cache at $godot_cache" >&2 +else + echo "Fetching the full Godot default-branch history without tags" >&2 + stage_dir=$(mktemp -d "$cache_dir/.godot-fetch.XXXXXX") + git clone --quiet --no-tags --single-branch "$godot_url" "$stage_dir/repo" + git -C "$stage_dir/repo" checkout --quiet --detach "$godot_revision" + validate_single_branch_cache "$stage_dir/repo" "$godot_revision" Godot + publish_cache "$stage_dir" "$godot_cache" Godot "$godot_revision" + echo "Cached Godot at $godot_cache" >&2 +fi + +if [[ -e $gcrypt_cache || -L $gcrypt_cache ]]; then + validate_single_branch_cache "$gcrypt_cache" "$gcrypt_revision" git-remote-gcrypt + [[ -x $gcrypt_cache/git-remote-gcrypt ]] || { + echo "git-remote-gcrypt executable is missing in $gcrypt_cache" >&2 + exit 1 + } + echo "Reusing git-remote-gcrypt cache at $gcrypt_cache" >&2 +else + echo "Fetching the pinned git-remote-gcrypt history" >&2 + stage_dir=$(mktemp -d "$cache_dir/.gcrypt-fetch.XXXXXX") + git clone --quiet --no-tags --single-branch "$gcrypt_url" "$stage_dir/repo" + git -C "$stage_dir/repo" checkout --quiet --detach "$gcrypt_revision" + validate_single_branch_cache "$stage_dir/repo" "$gcrypt_revision" git-remote-gcrypt + [[ -x $stage_dir/repo/git-remote-gcrypt ]] || { + echo "git-remote-gcrypt executable is missing at the pinned revision" >&2 + exit 1 + } + publish_cache "$stage_dir" "$gcrypt_cache" git-remote-gcrypt "$gcrypt_revision" + echo "Cached git-remote-gcrypt at $gcrypt_cache" >&2 +fi + +[[ $(git -C "$godot_cache" rev-parse HEAD) == "$godot_revision" ]] +[[ $(git -C "$gcrypt_cache" rev-parse HEAD) == "$gcrypt_revision" ]] +[[ -x $gcrypt_cache/git-remote-gcrypt ]] + +if [[ -n ${BENCH_RESULTS_DIR:-} ]]; then + results_dir=$BENCH_RESULTS_DIR + [[ $results_dir == /* ]] || results_dir="$PWD/$results_dir" + mkdir -p "$(dirname "$results_dir")" + mkdir "$results_dir" +else + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + host_short=$(hostname | awk -F. '{print $1}') + host_short=$(printf '%s' "$host_short" | tr -c '[:alnum:]_.-' '_') + results_dir="$project_root/bench-results/$timestamp-$host_short" + mkdir -p "$(dirname "$results_dir")" + mkdir "$results_dir" +fi +results_dir=$(cd "$results_dir" && pwd -P) +results_created=true + +echo "Running $rounds round(s) with $tiny_commits tiny commit(s)" >&2 +BENCH_GCRYPT_TINY_COMMITS=$gcrypt_tiny_commits \ + "$project_root/scripts/benchmark-git-backend.sh" \ + "$godot_cache" \ + "$gcrypt_cache" \ + "$godot_revision" \ + "$gcrypt_revision" \ + "$results_dir" >/dev/null + +os_name= +os_version= +os_build= +case $os in + Darwin) + os_name=$(sw_vers -productName) + os_version=$(sw_vers -productVersion) + os_build=$(sw_vers -buildVersion) + cpu_model=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true) + if [[ -z $cpu_model ]]; then + cpu_model=$(sysctl -n hw.model 2>/dev/null || uname -m) + fi + core_count=$(sysctl -n hw.ncpu) + ram_bytes=$(sysctl -n hw.memsize) + ;; + Linux) + os_name=$(awk -F= '$1 == "NAME" { gsub(/^"|"$/, "", $2); print $2; exit }' /etc/os-release 2>/dev/null || true) + os_version=$(awk -F= '$1 == "VERSION_ID" { gsub(/^"|"$/, "", $2); print $2; exit }' /etc/os-release 2>/dev/null || true) + [[ -n $os_name ]] || os_name=Linux + [[ -n $os_version ]] || os_version=$(uname -r) + cpu_model= + if command -v lscpu >/dev/null 2>&1; then + cpu_model=$(lscpu 2>/dev/null | awk -F: ' + { + label = tolower($1) + value = $2 + sub(/^[[:space:]]+/, "", label) + sub(/[[:space:]]+$/, "", label) + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + if (label == "model name" && model == "") model = value + if (label == "vendor id" && vendor == "") vendor = value + } + END { + if (model != "") print model + else if (vendor != "") print vendor + } + ' || true) + fi + if [[ -z $cpu_model ]]; then + cpu_model=$(awk -F: ' + { + label = tolower($1) + value = $2 + sub(/^[[:space:]]+/, "", label) + sub(/[[:space:]]+$/, "", label) + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + if (label == "model name" && model_name == "") model_name = value + if (label == "hardware" && hardware == "") hardware = value + if (label == "processor" && processor == "") processor = value + if (label == "cpu implementer" && implementer == "") implementer = value + if (label == "cpu part" && part == "") part = value + } + END { + if (model_name != "") { + print model_name + } else if (hardware != "") { + print hardware + } else if (implementer != "" && part != "") { + printf "CPU implementer %s, CPU part %s\n", implementer, part + } else if (processor != "") { + print processor + } else if (implementer != "") { + printf "CPU implementer %s\n", implementer + } else if (part != "") { + printf "CPU part %s\n", part + } + } + ' /proc/cpuinfo 2>/dev/null || true) + fi + [[ -n $cpu_model ]] || cpu_model=$(uname -m) + core_count=$(getconf _NPROCESSORS_ONLN 2>/dev/null || true) + if [[ -z $core_count ]]; then + core_count=$(awk '/^processor[[:space:]]*:/ { count++ } END { print count + 0 }' /proc/cpuinfo) + fi + ram_bytes=$(awk '/^MemTotal:/ { printf "%.0f", $2 * 1024; exit }' /proc/meminfo) + ;; +esac +kernel=$(uname -r) + +json_quote() { + local value=$1 + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + value=${value//$'\t'/\\t} + printf '"%s"' "$value" +} + +{ + printf '{\n' + printf ' "os_name": %s,\n' "$(json_quote "$os_name")" + printf ' "os_version": %s,\n' "$(json_quote "$os_version")" + printf ' "os_build": %s,\n' "$(json_quote "$os_build")" + printf ' "kernel": %s,\n' "$(json_quote "$kernel")" + printf ' "cpu_model": %s,\n' "$(json_quote "$cpu_model")" + printf ' "core_count": %s,\n' "$core_count" + printf ' "ram_bytes": %s,\n' "$ram_bytes" + printf ' "work_filesystem": %s,\n' "$(json_quote "$filesystem")" + printf ' "git_version": %s,\n' "$(json_quote "$git_version")" + printf ' "gpg_version": %s,\n' "$(json_quote "$gpg_version")" + printf ' "rustc_version": %s,\n' "$(json_quote "$rustc_version")" + printf ' "cargo_version": %s,\n' "$(json_quote "$cargo_version")" + printf ' "godot_revision": %s,\n' "$(json_quote "$godot_revision")" + printf ' "gcrypt_revision": %s,\n' "$(json_quote "$gcrypt_revision")" + printf ' "e2ee_revision": %s,\n' "$(json_quote "$e2ee_revision")" + printf ' "e2ee_uncommitted_changes": %s,\n' "$e2ee_dirty" + printf ' "benchmark_parameters": {\n' + printf ' "rounds": %s,\n' "$rounds" + printf ' "tiny_commits": %s,\n' "$tiny_commits" + printf ' "gcrypt_tiny_commits": %s,\n' "$gcrypt_tiny_commits" + printf ' "e2ee_only": %s,\n' "$([[ $e2ee_only == 1 ]] && echo true || echo false)" + printf ' "carrier_attr_tree": %s,\n' "$([[ $carrier_attr_tree == 1 ]] && echo true || echo false)" + printf ' "trace": %s,\n' "$([[ $trace == 1 ]] && echo true || echo false)" + printf ' "skip_build": %s,\n' "$([[ $skip_build == 1 ]] && echo true || echo false)" + printf ' "keep_work": %s\n' "$([[ $keep_work == 1 ]] && echo true || echo false)" + printf ' }\n' + printf '}\n' +} >"$results_dir/environment.json" + +medians_tsv="$results_dir/medians.tsv" +metric() { + local transport=$1 phase=$2 iteration=$3 column=$4 + awk -F '\t' -v t="$transport" -v p="$phase" -v i="$iteration" -v c="$column" \ + '$1 == t && $2 == p && $3 == i { print $c; exit }' "$medians_tsv" +} +total_client_disk_bytes() { + local transport=$1 git_bytes auxiliary_bytes + git_bytes=$(metric "$transport" fresh_fetch all 9) + auxiliary_bytes=$(metric "$transport" fresh_fetch all 11) + if [[ -z $git_bytes || $git_bytes == - ]]; then + printf '-' + return + fi + [[ -n $auxiliary_bytes && $auxiliary_bytes != - ]] || auxiliary_bytes=0 + awk -v git="$git_bytes" -v auxiliary="$auxiliary_bytes" 'BEGIN { printf "%.0f", git + auxiliary }' +} +format_initial_time() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { printf "%.1f s", n + 0 }'; fi +} +format_operation_time() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { if (n < 1) printf "%.2f s", n + 0; else printf "%.1f s", n + 0 }'; fi +} +format_seconds_two() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { printf "%.2f s", n + 0 }'; fi +} +format_gib() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { printf "%.2f GiB", (n + 0) / 1073741824 }'; fi +} +format_mib() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" ' + function grouped(value, digits, result) { + digits = sprintf("%.0f", value) + while (length(digits) > 3) { + result = "," substr(digits, length(digits) - 2) result + digits = substr(digits, 1, length(digits) - 3) + } + return digits result + } + BEGIN { printf "%s MiB", grouped((n + 0) / 1048576) } + '; fi +} +format_sent_bytes() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { if (n >= 1000000) printf "%.0f MB", n / 1000000; else printf "%.1f KB", n / 1000 }'; fi +} + +if [[ $e2ee_only == 1 ]]; then + common_pushes=$tiny_commits +else + common_pushes=$gcrypt_tiny_commits +fi +summary_file="$results_dir/summary.md" +{ + printf '# Benchmark summary\n\n' + printf 'Medians across %s fresh round(s). Tiny commit and update rows cover %s commit(s) per round.\n\n' "$rounds" "$tiny_commits" + printf '| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` |\n' + printf '|---|---:|---:|---:|\n' + printf '| Initial encryption (to a local directory) | %s | %s | %s |\n' \ + "$(format_initial_time "$(metric plain initial_encryption all 4)")" \ + "$(format_initial_time "$(metric gcrypt initial_encryption all 4)")" \ + "$(format_initial_time "$(metric e2ee initial_encryption all 4)")" + printf '| Initial push | %s | %s | %s |\n' \ + "$(format_initial_time "$(metric plain initial_push all 4)")" \ + "$(format_initial_time "$(metric gcrypt initial_push all 4)")" \ + "$(format_initial_time "$(metric e2ee initial_push all 4)")" + printf '| Fresh fetch | %s | %s | %s |\n' \ + "$(format_initial_time "$(metric plain fresh_fetch all 4)")" \ + "$(format_initial_time "$(metric gcrypt fresh_fetch all 4)")" \ + "$(format_initial_time "$(metric e2ee fresh_fetch all 4)")" + printf '| Tiny commit | %s | %s | %s |\n' \ + "$(format_seconds_two "$(metric plain tiny_commit all 4)")" \ + "$(format_seconds_two "$(metric gcrypt tiny_commit all 4)")" \ + "$(format_seconds_two "$(metric e2ee tiny_commit all 4)")" + printf '| Tiny push | %s | %s | %s |\n' \ + "$(format_operation_time "$(metric plain tiny_push all 4)")" \ + "$(format_operation_time "$(metric gcrypt tiny_push all 4)")" \ + "$(format_operation_time "$(metric e2ee tiny_push all 4)")" + printf '| Tiny update (fetch) | %s | %s | %s |\n' \ + "$(format_operation_time "$(metric plain tiny_update all 4)")" \ + "$(format_operation_time "$(metric gcrypt tiny_update all 4)")" \ + "$(format_operation_time "$(metric e2ee tiny_update all 4)")" + printf '| Data sent per tiny push | %s | %s | %s |\n' \ + "$(format_sent_bytes "$(metric plain tiny_push all 6)")" \ + "$(format_sent_bytes "$(metric gcrypt tiny_push all 6)")" \ + "$(format_sent_bytes "$(metric e2ee tiny_push all 6)")" + printf '| Peak memory, initial push | %s | %s | %s |\n' \ + "$(format_gib "$(metric plain initial_push all 5)")" \ + "$(format_gib "$(metric gcrypt initial_push all 5)")" \ + "$(format_gib "$(metric e2ee initial_push all 5)")" + printf '| Peak memory, tiny push | %s | %s | %s |\n' \ + "$(format_mib "$(metric plain tiny_push all 5)")" \ + "$(format_mib "$(metric gcrypt tiny_push all 5)")" \ + "$(format_mib "$(metric e2ee tiny_push all 5)")" + tiny_push_label="tiny pushes" + if (( common_pushes == 1 )); then + tiny_push_label="tiny push" + fi + printf '| Remote size after %s %s | %s | %s | %s |\n' "$common_pushes" "$tiny_push_label" \ + "$(format_mib "$(metric plain tiny_push "$common_pushes" 7)")" \ + "$(format_mib "$(metric gcrypt tiny_push "$common_pushes" 7)")" \ + "$(format_mib "$(metric e2ee tiny_push "$common_pushes" 7)")" + printf '| Total client disk after fresh fetch (.git + auxiliary cache/state) | %s | %s | %s |\n' \ + "$(format_mib "$(total_client_disk_bytes plain)")" \ + "$(format_mib "$(total_client_disk_bytes gcrypt)")" \ + "$(format_mib "$(total_client_disk_bytes e2ee)")" + if (( common_pushes != tiny_commits )); then + printf '\nThe remote-size row uses %s pushes because gcrypt ran %s tiny pushes.\n' "$common_pushes" "$gcrypt_tiny_commits" + fi +} >"$summary_file" + +run_complete=true +printf 'Results: %s\n\n' "$results_dir" +cat "$summary_file" diff --git a/src/crypto.rs b/src/crypto.rs index 470bb10..3beb615 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -1,11 +1,12 @@ use std::fs::{self, OpenOptions}; use std::io::{Read, Write}; use std::path::Path; +use std::time::Instant; +use aead_stream::{DecryptorBE32, EncryptorBE32, Nonce, StreamBE32}; use anyhow::{Context, Result, bail}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; -use chacha20poly1305::aead::stream::{DecryptorBE32, EncryptorBE32, Nonce, StreamBE32}; use chacha20poly1305::aead::{Aead, KeyInit, Payload}; use chacha20poly1305::{XChaCha20Poly1305, XNonce}; use ed25519_dalek::{Signature, Signer, SigningKey, VerifyingKey}; @@ -23,6 +24,8 @@ use sha2::{Digest, Sha256}; use subtle::ConstantTimeEq; use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; +use crate::trace; + const SYMMETRIC_MAGIC: &[u8; 8] = b"E2EES05\0"; const PACK_STREAM_MAGIC: &[u8; 8] = b"E2EEPK5\0"; const HPKE_INFO: &[u8] = b"git-remote-e2ee generation key v5"; @@ -340,9 +343,13 @@ pub fn seal_with_key(key: &[u8; 32], plaintext: &[u8], associated_data: &[u8]) - let cipher = XChaCha20Poly1305::new(key.into()); let mut nonce = [0_u8; 24]; OsRng.fill_bytes(&mut nonce); + let nonce: &XNonce = nonce + .as_slice() + .try_into() + .expect("XChaCha20-Poly1305 nonce has the fixed expected size"); let ciphertext = cipher .encrypt( - XNonce::from_slice(&nonce), + nonce, Payload { msg: plaintext, aad: associated_data, @@ -351,7 +358,7 @@ pub fn seal_with_key(key: &[u8; 32], plaintext: &[u8], associated_data: &[u8]) - .map_err(|_| anyhow::anyhow!("encryption failed"))?; let mut envelope = Vec::with_capacity(SYMMETRIC_MAGIC.len() + nonce.len() + ciphertext.len()); envelope.extend_from_slice(SYMMETRIC_MAGIC); - envelope.extend_from_slice(&nonce); + envelope.extend_from_slice(nonce); envelope.extend_from_slice(&ciphertext); Ok(envelope) } @@ -365,9 +372,12 @@ pub fn open_with_key(key: &[u8; 32], envelope: &[u8], associated_data: &[u8]) -> let nonce_start = SYMMETRIC_MAGIC.len(); let nonce_end = nonce_start + 24; let cipher = XChaCha20Poly1305::new(key.into()); + let nonce: &XNonce = envelope[nonce_start..nonce_end] + .try_into() + .expect("validated encrypted object has the fixed nonce size"); cipher .decrypt( - XNonce::from_slice(&envelope[nonce_start..nonce_end]), + nonce, Payload { msg: &envelope[nonce_end..], aad: associated_data, @@ -387,6 +397,8 @@ struct DigestWriter { inner: W, hasher: Sha256, count: u64, + stage_write: trace::Io, + sha256: trace::Io, } impl DigestWriter { @@ -395,6 +407,8 @@ impl DigestWriter { inner, hasher: Sha256::new(), count: 0, + stage_write: trace::Io::new("pack_ciphertext_stage_write"), + sha256: trace::Io::new("pack_ciphertext_sha256"), } } @@ -405,8 +419,17 @@ impl DigestWriter { impl Write for DigestWriter { fn write(&mut self, data: &[u8]) -> std::io::Result { + let write_started = self.stage_write.is_active().then(Instant::now); let written = self.inner.write(data)?; + if let Some(started) = write_started { + self.stage_write + .record(written, started.elapsed().as_nanos()); + } + let hash_started = self.sha256.is_active().then(Instant::now); self.hasher.update(&data[..written]); + if let Some(started) = hash_started { + self.sha256.record(written, started.elapsed().as_nanos()); + } self.count += written as u64; Ok(written) } @@ -425,11 +448,16 @@ fn pack_stream_aad(base_aad: &[u8], header: &[u8]) -> Vec { aad } -fn read_chunk(reader: &mut impl Read) -> Result> { +fn read_chunk(reader: &mut impl Read, input: &mut trace::Io) -> Result> { let mut chunk = vec![0_u8; PACK_STREAM_CHUNK_SIZE]; let mut filled = 0; while filled < chunk.len() { - match reader.read(&mut chunk[filled..]) { + let read_started = input.is_active().then(Instant::now); + let read_result = reader.read(&mut chunk[filled..]); + if let (Some(started), Ok(read)) = (read_started, &read_result) { + input.record(*read, started.elapsed().as_nanos()); + } + match read_result { Ok(0) => break, Ok(read) => filled += read, Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} @@ -446,6 +474,7 @@ pub fn seal_pack_stream( ciphertext: impl Write, base_aad: &[u8], ) -> Result { + trace::report_chacha20_backend(); let mut nonce = [0_u8; PACK_STREAM_NONCE_SIZE]; OsRng.fill_bytes(&mut nonce); let mut header = Vec::with_capacity(PACK_STREAM_HEADER_SIZE); @@ -454,19 +483,22 @@ pub fn seal_pack_stream( header.extend_from_slice(&nonce); let aad = pack_stream_aad(base_aad, &header); let stream_nonce = - Nonce::>::from_slice(&nonce); - let mut encryptor = EncryptorBE32::::new(key.into(), stream_nonce); + Nonce::>::try_from(nonce.as_slice()) + .expect("pack stream nonce has the fixed expected size"); + let mut encryptor = EncryptorBE32::::new(key.into(), &stream_nonce); let mut output = DigestWriter::new(ciphertext); + let mut input = trace::Io::new("pack_plaintext_pipe_read"); + let mut encryption = trace::Io::new("pack_stream_chacha20poly1305"); output.write_all(&header)?; - let mut current = read_chunk(&mut plaintext)?; + let mut current = read_chunk(&mut plaintext, &mut input)?; if current.is_empty() { bail!("cannot encrypt an empty pack stream") } let mut plaintext_size = 0_u64; let mut segments = 0_u64; loop { - let next = read_chunk(&mut plaintext)?; + let next = read_chunk(&mut plaintext, &mut input)?; plaintext_size = plaintext_size .checked_add(current.len() as u64) .context("pack plaintext size overflow")?; @@ -475,15 +507,23 @@ pub fn seal_pack_stream( bail!("pack stream exceeds segment counter limit") } if next.is_empty() { + let encrypt_started = encryption.is_active().then(Instant::now); encryptor .encrypt_last_in_place(&aad, &mut current) .map_err(|_| anyhow::anyhow!("pack stream encryption failed"))?; + if let Some(started) = encrypt_started { + encryption.record(current.len(), started.elapsed().as_nanos()); + } output.write_all(¤t)?; break; } + let encrypt_started = encryption.is_active().then(Instant::now); encryptor .encrypt_next_in_place(&aad, &mut current) .map_err(|_| anyhow::anyhow!("pack stream encryption failed"))?; + if let Some(started) = encrypt_started { + encryption.record(current.len(), started.elapsed().as_nanos()); + } output.write_all(¤t)?; current = next; } @@ -504,6 +544,7 @@ pub fn open_pack_stream( plaintext_size: u64, expected_id: &str, ) -> Result { + trace::report_chacha20_backend(); if plaintext_size == 0 { bail!("pack plaintext size must be nonzero") } @@ -539,12 +580,12 @@ pub fn open_pack_stream( hasher.update(header); let aad = pack_stream_aad(base_aad, &header); let nonce_start = chunk_size_start + 4; - let stream_nonce = Nonce::>::from_slice( - &header[nonce_start..], - ); + let stream_nonce = + Nonce::>::try_from(&header[nonce_start..]) + .expect("validated pack stream header has the fixed nonce size"); let mut decryptor = Some(DecryptorBE32::::new( key.into(), - stream_nonce, + &stream_nonce, )); let mut remaining = plaintext_size; let mut ciphertext_count = PACK_STREAM_HEADER_SIZE as u64; @@ -760,6 +801,42 @@ mod tests { } } + #[test] + fn pack_stream_be32_wire_format_matches_previous_release() { + let key = [0x42_u8; 32]; + let nonce = [0x24_u8; PACK_STREAM_NONCE_SIZE]; + let aad = b"git-remote-e2ee stream compatibility"; + let mut first = b"first old-format STREAM segment".to_vec(); + let mut last = b"last old-format STREAM segment".to_vec(); + let stream_nonce = + Nonce::>::try_from(nonce.as_slice()) + .unwrap(); + let mut encryptor = EncryptorBE32::::new((&key).into(), &stream_nonce); + encryptor.encrypt_next_in_place(aad, &mut first).unwrap(); + encryptor.encrypt_last_in_place(aad, &mut last).unwrap(); + + let expected = "4bcfaa062cd22c76da334339c5ba90a0fd2a8f369e18458da2cde4848d0a662e656fddd69b2937ea72d3da289c7fb79c1afdeeff4cc3e234f31ac5454a7592e96c4dc99aa1a5473eb4dfe4dd821365b48c4738512eaf6264ee9e373823"; + assert_eq!( + format!("{}{}", hex::encode(&first), hex::encode(&last)), + expected + ); + + let ciphertext = hex::decode(expected).unwrap(); + let first_ciphertext_size = b"first old-format STREAM segment".len() + PACK_STREAM_TAG_SIZE; + let (first_ciphertext, last_ciphertext) = ciphertext.split_at(first_ciphertext_size); + let mut first_ciphertext = first_ciphertext.to_vec(); + let mut last_ciphertext = last_ciphertext.to_vec(); + let mut decryptor = DecryptorBE32::::new((&key).into(), &stream_nonce); + decryptor + .decrypt_next_in_place(aad, &mut first_ciphertext) + .unwrap(); + decryptor + .decrypt_last_in_place(aad, &mut last_ciphertext) + .unwrap(); + assert_eq!(first_ciphertext, b"first old-format STREAM segment"); + assert_eq!(last_ciphertext, b"last old-format STREAM segment"); + } + #[test] fn pack_stream_is_randomized_and_context_bound() { let plaintext = pack_bytes(PACK_STREAM_CHUNK_SIZE + 1); diff --git a/src/git.rs b/src/git.rs index 80932f8..0219910 100644 --- a/src/git.rs +++ b/src/git.rs @@ -6,6 +6,8 @@ use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; use anyhow::{Context, Result, bail}; +use crate::trace; + pub fn ensure_repository(repo: &Path) -> Result<()> { let output = Command::new("git") .arg("-C") @@ -18,6 +20,38 @@ pub fn ensure_repository(repo: &Path) -> Result<()> { Ok(()) } +pub fn has_complete_object_graph(repo: &Path) -> bool { + let shallow = match Command::new("git") + .arg("-C") + .arg(repo) + .args(["rev-parse", "--is-shallow-repository"]) + .output() + { + Ok(output) if output.status.success() => output, + _ => return false, + }; + if String::from_utf8_lossy(&shallow.stdout).trim() != "false" { + return false; + } + + let partial = match Command::new("git") + .arg("-C") + .arg(repo) + .args([ + "config", + "--local", + "--name-only", + "--get-regexp", + r"^(extensions\.partialclone|remote\..*\.promisor)$", + ]) + .output() + { + Ok(output) => output, + Err(_) => return false, + }; + partial.status.code() == Some(1) && partial.stdout.is_empty() +} + pub fn resolve_ref(repo: &Path, reference: &str) -> Result { git_text(repo, &["rev-parse", "--verify", reference]) } @@ -88,6 +122,7 @@ pub struct PackSource { stdout: Option, stderr: File, finished: bool, + _timer: trace::Span, } impl Read for PackSource { @@ -128,6 +163,7 @@ pub fn start_incremental_pack( new_refs: &BTreeMap, old_refs: &BTreeMap, ) -> Result { + let timer = trace::Span::new("git_pack_objects_stream_lifetime"); let stderr = tempfile::tempfile().context("create pack-objects stderr file")?; let mut child = Command::new("git") .arg("-C") @@ -159,6 +195,7 @@ pub fn start_incremental_pack( stdout: Some(stdout), stderr, finished: false, + _timer: timer, }) } @@ -167,6 +204,7 @@ pub struct PackImporter { stdin: Option, stderr: File, finished: bool, + _timer: trace::Span, } impl Write for PackImporter { @@ -211,6 +249,7 @@ impl Drop for PackImporter { } pub fn start_pack_import(repo: &Path) -> Result { + let timer = trace::Span::new("git_index_pack"); let stderr = tempfile::tempfile().context("create index-pack stderr file")?; let mut child = Command::new("git") .arg("-C") @@ -226,6 +265,7 @@ pub fn start_pack_import(repo: &Path) -> Result { stdin: Some(stdin), stderr, finished: false, + _timer: timer, }) } @@ -243,6 +283,7 @@ pub fn ensure_refs_connected_since( refs: &BTreeMap, verified_refs: &BTreeMap, ) -> Result<()> { + let _timer = trace::Span::new("git_ref_connectivity_walk"); if refs.is_empty() { return Ok(()); } @@ -303,12 +344,15 @@ pub fn ensure_refs_connected_since( String::from_utf8_lossy(&output.stderr).trim() ) } - if String::from_utf8(output.stdout)? - .lines() - .any(|line| line.starts_with('?')) - { - bail!("fetched refs contain missing Git objects") + let output = String::from_utf8(output.stdout)?; + let mut checked_objects = 0; + for line in output.lines() { + checked_objects += 1; + if line.starts_with('?') { + bail!("fetched refs contain missing Git objects") + } } + trace::count("git_ref_connectivity_walk_objects", checked_objects); Ok(()) } @@ -436,3 +480,68 @@ fn git_text(repo: &Path, args: &[&str]) -> Result { } Ok(String::from_utf8(output.stdout)?.trim().to_owned()) } + +#[cfg(test)] +mod tests { + use super::has_complete_object_graph; + use std::fs; + use std::path::Path; + use std::process::Command; + + fn git(repo: &Path, args: &[&str]) { + let output = Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?}: {}", + args, + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn complete_object_graph_rejects_shallow_and_promisor_repositories() { + let temporary = tempfile::tempdir().unwrap(); + let full = temporary.path().join("full"); + fs::create_dir_all(&full).unwrap(); + let init = Command::new("git") + .args(["init", "--quiet", "--initial-branch=main"]) + .arg(&full) + .output() + .unwrap(); + assert!(init.status.success()); + git(&full, &["config", "user.name", "Test"]); + git(&full, &["config", "user.email", "test@example.invalid"]); + for contents in ["first", "second"] { + fs::write(full.join("file"), contents).unwrap(); + git(&full, &["add", "file"]); + git(&full, &["commit", "--quiet", "-m", contents]); + } + assert!(has_complete_object_graph(&full)); + + let shallow = temporary.path().join("shallow"); + let source_url = format!("file://{}", full.display()); + let clone = Command::new("git") + .args(["clone", "--quiet", "--depth=1", "--no-tags"]) + .arg(source_url) + .arg(&shallow) + .output() + .unwrap(); + assert!( + clone.status.success(), + "git clone: {}", + String::from_utf8_lossy(&clone.stderr) + ); + assert!(!has_complete_object_graph(&shallow)); + + git(&full, &["config", "remote.origin.promisor", "true"]); + assert!(!has_complete_object_graph(&full)); + git(&full, &["config", "--unset", "remote.origin.promisor"]); + git(&full, &["config", "extensions.partialClone", "origin"]); + assert!(!has_complete_object_graph(&full)); + } +} diff --git a/src/lib.rs b/src/lib.rs index 9139380..456cdf6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,3 +5,4 @@ mod persist; pub mod policy; pub mod repository; pub mod storage; +mod trace; diff --git a/src/repository.rs b/src/repository.rs index 5c8d387..8b1de5f 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -22,6 +22,7 @@ use crate::manifest::{ }; use crate::policy::{DeviceRecord, DeviceRoles, PolicyState}; use crate::storage::{HeadObservation, ObjectKind, Storage}; +use crate::trace; const MAX_RECOVERY_OUTER_COMMITS: usize = 2048; const MAX_RECOVERY_CANDIDATES: usize = 256; @@ -436,6 +437,7 @@ impl EncryptedRepository { SubkeyKind::Pack, ordinal, )?; + let pack_timer = trace::Span::new("inner_pack_encrypt_and_store"); let mut pack_source = git::start_incremental_pack(repo, &pack_refs, &exclusions)?; let mut pack_stage = self.storage.begin_object(ObjectKind::Pack)?; let sealed = seal_pack_stream( @@ -445,7 +447,10 @@ impl EncryptedRepository { &pack_aad(¤t.manifest.repository_root, generation, ordinal), )?; pack_source.finish()?; + let stage_finish_timer = trace::Span::new("inner_pack_stage_finish"); pack_stage.finish(&sealed.object_id)?; + drop(stage_finish_timer); + drop(pack_timer); new_packs.push(PackDescriptor { id: sealed.object_id, plaintext_size: sealed.plaintext_size, @@ -484,7 +489,12 @@ impl EncryptedRepository { next.clone(), )?; if let Some(state_path) = state_path.as_deref() { - self.pin_manifest_locked(state_path, &next_id, &next)?; + self.pin_manifest_locked( + state_path, + &next_id, + &next, + Some((repo, ¤t.manifest.refs)), + )?; } Ok(next_id) } @@ -702,6 +712,7 @@ impl EncryptedRepository { descriptor.ordinal, )?; let mut importer = git::start_pack_import(repo)?; + let decrypt_timer = trace::Span::new("pack_decrypt_to_index_pack"); open_pack_stream( &pack_key, encrypted, @@ -714,11 +725,14 @@ impl EncryptedRepository { descriptor.plaintext_size, &descriptor.id, )?; + drop(decrypt_timer); importer.finish()?; imported.insert(descriptor.id.clone()); } } + let connectivity_timer = trace::Span::new("inner_ref_connectivity_check"); git::ensure_refs_connected_since(repo, ¤t.manifest.refs, &state.verified_refs)?; + drop(connectivity_timer); write_client_state_locked( state_path, &imported, @@ -739,8 +753,18 @@ impl EncryptedRepository { state_path: &Path, head_id: &str, manifest: &Manifest, + published_from: Option<(&Path, &BTreeMap)>, ) -> Result<()> { - let state = read_client_state(state_path)?; + let mut state = read_client_state(state_path)?; + if let Some((repo, previous_refs)) = published_from + && state.verified_refs == *previous_refs + && git::has_complete_object_graph(repo) + { + // The completed local pack walk covers new reachability only beyond + // refs already verified by this client. Shallow and promisor repos + // can omit history, so they must not advance this frontier. + state.verified_refs = manifest.refs.clone(); + } write_published_client_state_locked(state_path, state, head_id, manifest) } @@ -1283,7 +1307,7 @@ impl EncryptedRepository { ManifestAuthorization::Writer, next.clone(), )?; - self.pin_manifest_locked(&state_path, &recovered_id, &next)?; + self.pin_manifest_locked(&state_path, &recovered_id, &next, None)?; report.published_manifest = Some(recovered_id); report.blocked_reason = None; Ok(report) @@ -1397,6 +1421,7 @@ impl EncryptedRepository { } fn current_chain(&self) -> Result> { + let _timer = trace::Span::new("manifest_chain_authentication"); let head = self .storage .read_head()? @@ -1550,6 +1575,7 @@ impl EncryptedRepository { authorization: ManifestAuthorization, manifest: Manifest, ) -> Result { + let manifest_timer = trace::Span::new("manifest_seal_and_self_verify"); match head.previous { Some(previous) => { manifest.validate_successor(&previous.id, &previous.manifest, policy)? @@ -1579,6 +1605,8 @@ impl EncryptedRepository { if opened.header.generation != manifest.generation { bail!("new manifest self-check changed generation") } + drop(manifest_timer); + let cas_timer = trace::Span::new("manifest_compare_and_swap_publish"); if let Some(observation) = head.observation { self.storage .compare_and_swap_observed_head(observation, &id)?; @@ -1586,6 +1614,7 @@ impl EncryptedRepository { self.storage .compare_and_swap_head(head.previous.map(|state| state.id.as_str()), &id)?; } + drop(cas_timer); Ok(id) } } diff --git a/src/storage.rs b/src/storage.rs index f72669d..3bce390 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,8 +1,8 @@ use std::collections::HashSet; use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; +use std::io::{BufRead, BufReader, Read, Write}; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Child, ChildStdin, Command, Stdio}; use std::sync::Mutex; use anyhow::{Context, Result, bail}; @@ -13,6 +13,7 @@ use sha2::{Digest, Sha256}; use thiserror::Error; use crate::crypto::object_id; +use crate::trace; #[derive(Debug, Error)] #[error("head changed concurrently (expected {expected:?}, actual {actual:?})")] @@ -440,6 +441,7 @@ fn verify_file_id(path: &Path, id: &str) -> Result<()> { const CARRIER_BRANCH: &str = "git-remote-e2ee"; const CARRIER_CHUNK_SIZE: usize = 32 * 1024 * 1024; +const CARRIER_ATTRIBUTES: &[u8] = b"e2ee/** -delta\n"; const CARRIER_CACHE_ENV: &str = "GIT_REMOTE_E2EE_CACHE_DIR"; const CARRIER_CACHE_FETCH_REF: &str = "refs/heads/git-remote-e2ee"; const MAX_RECOVERY_OUTER_COMMITS: usize = 2048; @@ -458,19 +460,27 @@ struct ChunkReader { paths: Vec, next: usize, current: Option, + trace: trace::Io, } impl Read for ChunkReader { fn read(&mut self, output: &mut [u8]) -> std::io::Result { + let started = self.trace.is_active().then(std::time::Instant::now); loop { if let Some(file) = &mut self.current { let read = file.read(output)?; if read != 0 { + self.trace.record( + read, + started.map_or(0, |started| started.elapsed().as_nanos()), + ); return Ok(read); } self.current = None; } if self.next == self.paths.len() { + self.trace + .record(0, started.map_or(0, |started| started.elapsed().as_nanos())); return Ok(0); } self.current = Some(File::open(&self.paths[self.next])?); @@ -479,6 +489,148 @@ impl Read for ChunkReader { } } +struct GitChunkReader { + child: Child, + input: Option, + output: BufReader, + chunks: Vec<(String, String)>, + next: usize, + remaining: u64, + finished: bool, + trace: trace::Io, +} + +impl GitChunkReader { + fn new(repo: &Path, chunks: Vec<(String, String)>) -> Result { + let mut child = carrier_git_command() + .arg("-C") + .arg(repo) + .args(["cat-file", "--batch"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + let input = child.stdin.take().context("open git cat-file stdin")?; + let output = child.stdout.take().context("open git cat-file stdout")?; + Ok(Self { + child, + input: Some(input), + output: BufReader::new(output), + chunks, + next: 0, + remaining: 0, + finished: false, + trace: trace::Io::new("carrier_object_read"), + }) + } + + fn start_next_chunk(&mut self) -> std::io::Result { + if self.next == self.chunks.len() { + return Ok(false); + } + let expected = &self.chunks[self.next].1; + let input = self + .input + .as_mut() + .expect("git cat-file input is open until all chunks are read"); + writeln!(input, "{expected}")?; + input.flush()?; + let mut header = String::new(); + if self.output.read_line(&mut header)? == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "git cat-file ended before returning a carrier chunk", + )); + } + let fields: Vec<_> = header.split_ascii_whitespace().collect(); + if fields.len() != 3 || fields[0] != expected || fields[1] != "blob" { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "git cat-file returned an invalid carrier chunk header", + )); + } + let size = fields[2] + .parse::() + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + let last = self.next + 1 == self.chunks.len(); + if (!last && size != CARRIER_CHUNK_SIZE as u64) + || (last && (size == 0 || size > CARRIER_CHUNK_SIZE as u64)) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "carrier object chunk has invalid size", + )); + } + self.next += 1; + self.remaining = size; + Ok(true) + } + + fn finish(&mut self) -> std::io::Result<()> { + self.input.take(); + let status = self.child.wait()?; + self.finished = true; + if !status.success() { + return Err(std::io::Error::other("git cat-file failed")); + } + Ok(()) + } +} + +impl Read for GitChunkReader { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + if output.is_empty() { + return Ok(0); + } + let started = self.trace.is_active().then(std::time::Instant::now); + if self.remaining == 0 { + if self.next == self.chunks.len() { + self.finish()?; + self.trace + .record(0, started.map_or(0, |started| started.elapsed().as_nanos())); + return Ok(0); + } + if !self.start_next_chunk()? { + return Ok(0); + } + } + let amount = output.len().min(self.remaining as usize); + let read = self.output.read(&mut output[..amount])?; + if read == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "git cat-file truncated a carrier chunk", + )); + } + self.remaining -= read as u64; + if self.remaining == 0 { + let mut trailer = [0_u8; 1]; + self.output.read_exact(&mut trailer)?; + if trailer[0] != b'\n' { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "git cat-file returned a malformed chunk trailer", + )); + } + } + self.trace.record( + read, + started.map_or(0, |started| started.elapsed().as_nanos()), + ); + Ok(read) + } +} + +impl Drop for GitChunkReader { + fn drop(&mut self) { + if !self.finished { + self.input.take(); + let _ = self.child.kill(); + let _ = self.child.wait(); + } + } +} + struct CarrierObjectStage { staging: PathBuf, root: PathBuf, @@ -486,6 +638,7 @@ struct CarrierObjectStage { current: Option, chunk_index: usize, chunk_len: usize, + trace: trace::Io, } impl CarrierObjectStage { @@ -516,6 +669,7 @@ impl CarrierObjectStage { impl Write for CarrierObjectStage { fn write(&mut self, mut data: &[u8]) -> std::io::Result { let original = data.len(); + let started = self.trace.is_active().then(std::time::Instant::now); while !data.is_empty() { self.open_chunk()?; let available = CARRIER_CHUNK_SIZE - self.chunk_len; @@ -530,6 +684,10 @@ impl Write for CarrierObjectStage { self.finish_chunk()?; } } + self.trace.record( + original, + started.map_or(0, |started| started.elapsed().as_nanos()), + ); Ok(original) } @@ -559,6 +717,7 @@ impl ObjectStageSink for CarrierObjectStage { paths, next: 0, current: None, + trace: trace::Io::new("carrier_object_read"), })? != id { bail!("object id collision for {id}") @@ -613,11 +772,16 @@ impl GitStorage { bail!("empty carrier Git remote") } + let lock_timer = trace::Span::new("carrier_cache_lock"); let mut cache = CarrierCache::lock(remote)?; + drop(lock_timer); + let fetch_timer = trace::Span::new("carrier_cache_fetch"); let mut base_commit = cache.refresh(remote)?; + drop(fetch_timer); let mut checkout = tempfile::Builder::new() .prefix("git-remote-e2ee-carrier-") .tempdir()?; + let checkout_timer = trace::Span::new("carrier_checkout_create"); if let Err(error) = create_carrier_checkout(checkout.path(), &cache.path, remote, &base_commit) { @@ -633,6 +797,7 @@ impl GitStorage { create_carrier_checkout(checkout.path(), &cache.path, remote, &base_commit) .context("create carrier checkout after rebuilding corrupt cache")?; } + drop(checkout_timer); drop(cache); Ok(Self { @@ -664,17 +829,33 @@ impl GitStorage { validate_id(&value)?; Ok(Some(value)) } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let Some(commit) = git_rev_parse(self.root(), "HEAD")? else { + return Ok(None); + }; + let Some(bytes) = self.head_bytes_at_commit(&commit)? else { + return Ok(None); + }; + let value = String::from_utf8(bytes)?.trim().to_owned(); + validate_id(&value)?; + Ok(Some(value)) + } Err(error) => Err(error.into()), } } fn local_head_bytes(&self) -> Result>> { - read_bounded_head_file(&self.root().join("e2ee/HEAD")) - } - - fn refresh_remote(&self) -> Result> { - CarrierCache::lock(&self.remote)?.refresh(&self.remote) + let path = self.root().join("e2ee/HEAD"); + match read_bounded_head_file(&path) { + Ok(Some(bytes)) => Ok(Some(bytes)), + Ok(None) => { + let Some(commit) = git_rev_parse(self.root(), "HEAD")? else { + return Ok(None); + }; + self.head_bytes_at_commit(&commit) + } + Err(error) => Err(error), + } } fn head_at_commit(&self, commit: Option<&str>) -> Result> { @@ -757,54 +938,7 @@ impl GitStorage { kind: ObjectKind, id: &str, ) -> Result>> { - validate_id(id)?; - let directory = format!("e2ee/{}/{}/{id}", kind.directory(), &id[..2]); - let output = carrier_git_command() - .arg("-C") - .arg(self.root()) - .args(["ls-tree", "-r", "-z", commit, "--", &directory]) - .output()?; - ensure_git_success(&output, "git ls-tree historical carrier object")?; - if output.stdout.is_empty() { - return Ok(None); - } - if output.stdout.len() > 8 * 1024 * 1024 { - bail!("carrier recovery object listing exceeds its size budget") - } - let mut chunks = Vec::new(); - for entry in output - .stdout - .split(|byte| *byte == 0) - .filter(|entry| !entry.is_empty()) - { - let tab = entry - .iter() - .position(|byte| *byte == b'\t') - .context("parse historical carrier tree entry")?; - let (metadata, path) = (&entry[..tab], &entry[tab + 1..]); - let metadata = std::str::from_utf8(metadata)?; - let path = std::str::from_utf8(path)?.to_owned(); - let mut fields = metadata.split_ascii_whitespace(); - if fields.next() != Some("100644") || fields.next() != Some("blob") { - bail!("historical carrier object contains a non-regular blob") - } - let blob_id = fields.next().context("historical carrier blob has no id")?; - if fields.next().is_some() || !path.starts_with(&format!("{directory}/")) { - bail!("unexpected historical carrier object path") - } - chunks.push((path, blob_id.to_owned())); - } - chunks.sort_by(|left, right| left.0.cmp(&right.0)); - if chunks.is_empty() || chunks.len() > 4096 { - bail!("historical carrier object has an invalid chunk count") - } - for (index, (path, _)) in chunks.iter().enumerate() { - let expected = format!("{directory}/{index:08}"); - if path != &expected { - bail!("historical carrier object chunks are not a dense canonical sequence") - } - } - Ok(Some(chunks)) + carrier_chunk_entries(self.root(), commit, kind, id) } fn object_at_commit( @@ -890,14 +1024,27 @@ impl GitStorage { Ok(true) } - fn publish_checkout_head(&self, next: &str, expected: Option) -> Result<()> { + fn publish_checkout_head( + &self, + next: &str, + expected: Option, + cache: &mut CarrierCache, + ) -> Result<()> { fs::create_dir_all(self.root().join("e2ee"))?; fs::write(self.root().join("e2ee/HEAD"), format!("{next}\n"))?; - git_command(self.root(), &["add", "e2ee"])?; - git_command( + let add_timer = trace::Span::new("carrier_git_add"); + stage_carrier_checkout(self.root(), &cache.path.join("objects"))?; + drop(add_timer); + let commit_timer = trace::Span::new("carrier_git_commit"); + git_command_with_object_directory( self.root(), + &cache.path.join("objects"), &["commit", "--quiet", "-m", "git-remote-e2ee storage update"], )?; + let new_commit = + git_rev_parse(self.root(), "HEAD")?.context("carrier commit was not created")?; + drop(commit_timer); + let push_timer = trace::Span::new("carrier_git_push"); let push = carrier_git_command() .arg("-C") .arg(self.root()) @@ -905,13 +1052,15 @@ impl GitStorage { .arg(format!("HEAD:refs/heads/{CARRIER_BRANCH}")) .output()?; if !push.status.success() { - let latest = self.refresh_remote()?; + let latest = cache.refresh(&self.remote)?; return Err(CasConflict { expected, actual: self.head_at_commit(latest.as_deref())?, } .into()); } + drop(push_timer); + cache.update_published_tip(&new_commit)?; Ok(()) } } @@ -929,16 +1078,28 @@ impl Storage for GitStorage { current: None, chunk_index: 0, chunk_len: 0, + trace: trace::Io::new("carrier_stage_write"), })) } fn open_object(&self, kind: ObjectKind, id: &str) -> Result> { - let paths = validated_chunk_paths(&self.object_directory(kind, id)?)?; - Ok(Box::new(ChunkReader { - paths, - next: 0, - current: None, - })) + let directory = self.object_directory(kind, id)?; + if directory.exists() { + let paths = validated_chunk_paths(&directory)?; + return Ok(Box::new(ChunkReader { + paths, + next: 0, + current: None, + trace: trace::Io::new("carrier_object_read"), + })); + } + let Some(commit) = git_rev_parse(self.root(), "HEAD")? else { + bail!("carrier object {id} is unavailable") + }; + let chunks = self + .chunk_entries_at(&commit, kind, id)? + .with_context(|| format!("carrier object {id} is unavailable"))?; + Ok(Box::new(GitChunkReader::new(self.root(), chunks)?)) } fn read_head(&self) -> Result> { @@ -953,7 +1114,10 @@ impl Storage for GitStorage { .state .lock() .map_err(|_| anyhow::anyhow!("carrier state lock poisoned"))?; - let remote_tip = self.refresh_remote()?; + let mut cache = CarrierCache::lock(&self.remote)?; + let refresh_timer = trace::Span::new("carrier_cas_refresh"); + let remote_tip = cache.refresh(&self.remote)?; + drop(refresh_timer); if remote_tip != state.base_commit { return Err(CasConflict { expected: expected.map(ToOwned::to_owned), @@ -972,13 +1136,19 @@ impl Storage for GitStorage { fs::create_dir_all(self.root().join("e2ee"))?; fs::write(self.root().join("e2ee/HEAD"), format!("{next}\n"))?; - git_command(self.root(), &["add", "e2ee"])?; - git_command( + let add_timer = trace::Span::new("carrier_git_add"); + stage_carrier_checkout(self.root(), &cache.path.join("objects"))?; + drop(add_timer); + let commit_timer = trace::Span::new("carrier_git_commit"); + git_command_with_object_directory( self.root(), + &cache.path.join("objects"), &["commit", "--quiet", "-m", "git-remote-e2ee storage update"], )?; + drop(commit_timer); let new_commit = git_rev_parse(self.root(), "HEAD")?.context("carrier commit was not created")?; + let push_timer = trace::Span::new("carrier_git_push"); let push = carrier_git_command() .arg("-C") .arg(self.root()) @@ -986,13 +1156,15 @@ impl Storage for GitStorage { .arg(format!("HEAD:refs/heads/{CARRIER_BRANCH}")) .output()?; if !push.status.success() { - let latest = self.refresh_remote()?; + let latest = cache.refresh(&self.remote)?; return Err(CasConflict { expected: expected.map(ToOwned::to_owned), actual: self.head_at_commit(latest.as_deref())?, } .into()); } + drop(push_timer); + cache.update_published_tip(&new_commit)?; state.base_commit = Some(new_commit); Ok(()) } @@ -1032,7 +1204,8 @@ impl Storage for GitStorage { if observed.token != expected_tip.as_deref().unwrap_or_default().as_bytes() { bail!("recovery storage token does not match the opened carrier tip") } - let remote_tip = self.refresh_remote()?; + let mut cache = CarrierCache::lock(&self.remote)?; + let remote_tip = cache.refresh(&self.remote)?; if remote_tip != expected_tip { return Err(CasConflict { expected: observed.head_id.clone(), @@ -1054,7 +1227,7 @@ impl Storage for GitStorage { } .into()); } - self.publish_checkout_head(next, observed.head_id.clone())?; + self.publish_checkout_head(next, observed.head_id.clone(), &mut cache)?; let new_commit = git_rev_parse(self.root(), "HEAD")?.context("carrier commit was not created")?; state.base_commit = Some(new_commit); @@ -1118,18 +1291,62 @@ impl Storage for GitStorage { } fn prepare_recovery(&self, manifest_ids: &[String], pack_ids: &[String]) -> Result<()> { - retain_carrier_objects( - &self.root().join("e2ee/manifests"), - &manifest_ids.iter().map(String::as_str).collect(), - )?; - retain_carrier_objects( - &self.root().join("e2ee/objects"), - &pack_ids.iter().map(String::as_str).collect(), - )?; + let manifests: HashSet<&str> = manifest_ids.iter().map(String::as_str).collect(); + let packs: HashSet<&str> = pack_ids.iter().map(String::as_str).collect(); + retain_carrier_objects(&self.root().join("e2ee/manifests"), &manifests)?; + retain_carrier_objects(&self.root().join("e2ee/objects"), &packs)?; + remove_unretained_index_objects(self.root(), ObjectKind::Manifest, &manifests)?; + remove_unretained_index_objects(self.root(), ObjectKind::Pack, &packs)?; Ok(()) } } +fn remove_unretained_index_objects( + repo: &Path, + kind: ObjectKind, + keep: &HashSet<&str>, +) -> Result<()> { + let tree = format!("e2ee/{}", kind.directory()); + let output = carrier_git_command() + .arg("-C") + .arg(repo) + .args(["ls-files", "-z", "--", &tree]) + .output()?; + ensure_git_success(&output, "git ls-files carrier recovery objects")?; + let mut remove = Vec::new(); + for path in output + .stdout + .split(|byte| *byte == 0) + .filter(|path| !path.is_empty()) + { + let path = std::str::from_utf8(path)?; + let mut fields = path.split('/'); + if fields.next() != Some("e2ee") || fields.next() != Some(kind.directory()) { + bail!("unexpected carrier index path during recovery") + } + let _prefix = fields.next().context("carrier index path has no prefix")?; + let id = fields + .next() + .context("carrier index path has no object id")?; + let _chunk = fields.next().context("carrier index path has no chunk")?; + if fields.next().is_some() { + bail!("unexpected carrier index path depth during recovery") + } + if !keep.contains(id) { + remove.extend_from_slice(path.as_bytes()); + remove.push(0); + } + } + if !remove.is_empty() { + git_command_with_input( + repo, + &["update-index", "--force-remove", "-z", "--stdin"], + &remove, + )?; + } + Ok(()) +} + fn retain_carrier_objects(root: &Path, keep: &HashSet<&str>) -> Result<()> { let entries = match fs::read_dir(root) { Ok(entries) => entries.collect::>>()?, @@ -1159,6 +1376,62 @@ fn retain_carrier_objects(root: &Path, keep: &HashSet<&str>) -> Result<()> { Ok(()) } +fn carrier_chunk_entries( + repo: &Path, + commit: &str, + kind: ObjectKind, + id: &str, +) -> Result>> { + validate_id(id)?; + let directory = format!("e2ee/{}/{}/{id}", kind.directory(), &id[..2]); + let output = carrier_git_command() + .arg("-C") + .arg(repo) + .args(["ls-tree", "-r", "-z", commit, "--", &directory]) + .output()?; + ensure_git_success(&output, "git ls-tree carrier object")?; + if output.stdout.is_empty() { + return Ok(None); + } + if output.stdout.len() > 8 * 1024 * 1024 { + bail!("carrier object listing exceeds its size budget") + } + let mut chunks = Vec::new(); + for entry in output + .stdout + .split(|byte| *byte == 0) + .filter(|entry| !entry.is_empty()) + { + let tab = entry + .iter() + .position(|byte| *byte == b'\t') + .context("parse carrier tree entry")?; + let (metadata, path) = (&entry[..tab], &entry[tab + 1..]); + let metadata = std::str::from_utf8(metadata)?; + let path = std::str::from_utf8(path)?.to_owned(); + let mut fields = metadata.split_ascii_whitespace(); + if fields.next() != Some("100644") || fields.next() != Some("blob") { + bail!("carrier object contains a non-regular blob") + } + let blob_id = fields.next().context("carrier blob has no id")?; + if fields.next().is_some() || !path.starts_with(&format!("{directory}/")) { + bail!("unexpected carrier object path") + } + chunks.push((path, blob_id.to_owned())); + } + chunks.sort_by(|left, right| left.0.cmp(&right.0)); + if chunks.is_empty() || chunks.len() > 4096 { + bail!("carrier object has an invalid chunk count") + } + for (index, (path, _)) in chunks.iter().enumerate() { + let expected = format!("{directory}/{index:08}"); + if path != &expected { + bail!("carrier object chunks are not a dense canonical sequence") + } + } + Ok(Some(chunks)) +} + struct CarrierCache { remote_dir: PathBuf, path: PathBuf, @@ -1181,8 +1454,10 @@ impl CarrierCache { .write(true) .open(&lock_path) .with_context(|| format!("open carrier cache lock {}", lock_path.display()))?; + let wait_timer = trace::Span::new("carrier_cache_lock_wait"); lock.lock_exclusive() .with_context(|| format!("lock carrier cache {}", lock_path.display()))?; + drop(wait_timer); let remote_dir = root.join(&hash); Ok(Self { remote_dir, @@ -1192,18 +1467,29 @@ impl CarrierCache { } fn refresh(&mut self, remote: &str) -> Result> { + let initialize_timer = trace::Span::new("carrier_cache_initialize"); self.ensure_initialized()?; + drop(initialize_timer); + let fetch_timer = trace::Span::new("carrier_cache_fetch_remote"); match fetch_carrier_branch(&self.path, remote) { - Ok(tip) => Ok(tip), + Ok(tip) => { + drop(fetch_timer); + Ok(tip) + } Err(fetch_error) if self.is_corrupt()? => { self.rebuild()?; - fetch_carrier_branch(&self.path, remote).with_context(|| { + let result = fetch_carrier_branch(&self.path, remote).with_context(|| { format!( "fetch carrier branch after rebuilding corrupt cache (initial fetch failed: {fetch_error:#})" ) - }) + }); + drop(fetch_timer); + result + } + Err(fetch_error) => { + drop(fetch_timer); + Err(fetch_error) } - Err(fetch_error) => Err(fetch_error), } } @@ -1297,6 +1583,10 @@ impl CarrierCache { fs::rename(&temporary, ¤t)?; Ok(()) } + + fn update_published_tip(&self, commit: &str) -> Result<()> { + cache_git_command(&self.path, &["update-ref", CARRIER_CACHE_FETCH_REF, commit]) + } } fn valid_cache_generation(value: &str) -> bool { @@ -1374,6 +1664,21 @@ fn lexically_normalize(path: &Path) -> PathBuf { fn configure_cache(cache: &Path) -> Result<()> { cache_git_command(cache, &["config", "gc.auto", "0"])?; cache_git_command(cache, &["config", "maintenance.auto", "false"])?; + configure_git_pack_settings(|key, value| cache_git_command(cache, &["config", key, value]))?; + Ok(()) +} + +fn configure_git_pack_settings(mut configure: impl FnMut(&str, &str) -> Result<()>) -> Result<()> { + for (key, value) in [ + ("core.compression", "0"), + ("core.looseCompression", "0"), + ("pack.compression", "0"), + ("pack.window", "0"), + ("pack.depth", "0"), + ("core.bigFileThreshold", "1m"), + ] { + configure(key, value)?; + } Ok(()) } @@ -1404,6 +1709,7 @@ fn create_carrier_checkout( remote: &str, base_commit: &Option, ) -> Result<()> { + let _timer = trace::Span::new("carrier_checkout_git_setup"); let output = carrier_git_command() .arg("-C") .arg(checkout) @@ -1423,6 +1729,7 @@ fn create_carrier_checkout( )?; git_command(checkout, &["config", "gc.auto", "0"])?; git_command(checkout, &["config", "maintenance.auto", "false"])?; + configure_git_pack_settings(|key, value| git_command(checkout, &["config", key, value]))?; git_command(checkout, &["config", "user.name", "git-remote-e2ee"])?; git_command( checkout, @@ -1434,10 +1741,10 @@ fn create_carrier_checkout( match base_commit { Some(commit) => { git_command(checkout, &["update-ref", &remote_ref, commit])?; - git_command( - checkout, - &["checkout", "--quiet", "-B", CARRIER_BRANCH, &remote_ref], - )?; + let local_ref = format!("refs/heads/{CARRIER_BRANCH}"); + git_command(checkout, &["update-ref", &local_ref, commit])?; + git_command(checkout, &["symbolic-ref", "HEAD", &local_ref])?; + git_command(checkout, &["read-tree", commit])?; } None => { git_command( @@ -1446,6 +1753,7 @@ fn create_carrier_checkout( )?; } } + fs::write(checkout.join(".gitattributes"), CARRIER_ATTRIBUTES)?; Ok(()) } @@ -1501,6 +1809,126 @@ fn git_command(repo: &Path, args: &[&str]) -> Result<()> { Ok(()) } +fn git_command_with_input(repo: &Path, args: &[&str], input: &[u8]) -> Result<()> { + let mut child = carrier_git_command() + .arg("-C") + .arg(repo) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + child + .stdin + .take() + .context("open git stdin")? + .write_all(input)?; + let output = child.wait_with_output()?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + } + Ok(()) +} + +fn stage_carrier_checkout(repo: &Path, object_directory: &Path) -> Result<()> { + fn collect(root: &Path, path: &Path, output: &mut Vec) -> Result<()> { + let entry = fs::symlink_metadata(path)?; + if entry.file_type().is_dir() { + for child in fs::read_dir(path)? { + collect(root, &child?.path(), output)?; + } + } else if entry.file_type().is_file() { + let relative = path + .strip_prefix(root)? + .to_str() + .context("carrier checkout path is not UTF-8")?; + if relative.contains('\n') || relative.contains('\0') { + bail!("carrier checkout path contains a forbidden character") + } + output.extend_from_slice(relative.as_bytes()); + output.push(0); + } else { + bail!("carrier checkout contains a non-regular staged path") + } + Ok(()) + } + + let mut paths = Vec::new(); + let attributes = repo.join(".gitattributes"); + if attributes.exists() { + collect(repo, &attributes, &mut paths)?; + } + let encrypted = repo.join("e2ee"); + if encrypted.exists() { + collect(repo, &encrypted, &mut paths)?; + } + if paths.is_empty() { + bail!("carrier publication has no staged files") + } + git_command_with_object_directory_input( + repo, + object_directory, + &["add", "-A", "--pathspec-from-file=-", "--pathspec-file-nul"], + &paths, + ) +} + +fn git_command_with_object_directory( + repo: &Path, + object_directory: &Path, + args: &[&str], +) -> Result<()> { + let output = carrier_git_command() + .arg("-C") + .arg(repo) + .env("GIT_OBJECT_DIRECTORY", object_directory) + .args(args) + .output()?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + } + Ok(()) +} + +fn git_command_with_object_directory_input( + repo: &Path, + object_directory: &Path, + args: &[&str], + input: &[u8], +) -> Result<()> { + let mut child = carrier_git_command() + .arg("-C") + .arg(repo) + .env("GIT_OBJECT_DIRECTORY", object_directory) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + child + .stdin + .take() + .context("open git pathspec stdin")? + .write_all(input)?; + let output = child.wait_with_output()?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + } + Ok(()) +} + fn carrier_git_command() -> Command { let mut command = Command::new("git"); for variable in [ @@ -1657,7 +2085,8 @@ mod tests { reader_id(ChunkReader { paths, next: 0, - current: None + current: None, + trace: trace::Io::new("carrier_object_read"), }) .unwrap(), object_id(b"only") diff --git a/src/trace.rs b/src/trace.rs new file mode 100644 index 0000000..bce9f66 --- /dev/null +++ b/src/trace.rs @@ -0,0 +1,110 @@ +use std::sync::OnceLock; +use std::time::Instant; + +fn enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var("GIT_REMOTE_E2EE_TRACE").is_ok_and(|value| value == "1")) +} + +pub(crate) fn report_chacha20_backend() { + if !enabled() { + return; + } + + static REPORTED: OnceLock<()> = OnceLock::new(); + REPORTED.get_or_init(|| { + eprintln!( + "git-remote-e2ee trace chacha20_backend={}", + chacha20_backend() + ); + }); +} + +pub(crate) fn count(name: &'static str, value: usize) { + if enabled() { + eprintln!("git-remote-e2ee trace count={name} value={value}"); + } +} + +#[allow(unexpected_cfgs)] +fn chacha20_backend() -> &'static str { + if cfg!(chacha20_backend = "soft") { + "portable" + } else if cfg!(all(target_arch = "aarch64", target_feature = "neon")) { + "aarch64-neon" + } else if cfg!(target_arch = "aarch64") { + "aarch64-portable" + } else if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + "x86-runtime-dispatch" + } else { + "portable" + } +} + +pub(crate) struct Span { + name: &'static str, + started: Option, +} + +impl Span { + pub(crate) fn new(name: &'static str) -> Self { + Self { + name, + started: enabled().then(Instant::now), + } + } +} + +impl Drop for Span { + fn drop(&mut self) { + if let Some(started) = self.started { + eprintln!( + "git-remote-e2ee trace phase={} elapsed_ms={}", + self.name, + started.elapsed().as_millis() + ); + } + } +} + +pub(crate) struct Io { + name: &'static str, + started: Option, + elapsed_nanos: u128, + bytes: u64, +} + +impl Io { + pub(crate) fn new(name: &'static str) -> Self { + Self { + name, + started: enabled().then(Instant::now), + elapsed_nanos: 0, + bytes: 0, + } + } + + pub(crate) fn record(&mut self, bytes: usize, elapsed_nanos: u128) { + if self.started.is_some() { + self.bytes = self.bytes.saturating_add(bytes as u64); + self.elapsed_nanos = self.elapsed_nanos.saturating_add(elapsed_nanos); + } + } + + pub(crate) fn is_active(&self) -> bool { + self.started.is_some() + } +} + +impl Drop for Io { + fn drop(&mut self) { + if self.started.is_some() { + eprintln!( + "git-remote-e2ee trace io={} elapsed_ms={} bytes={}", + self.name, + self.elapsed_nanos / 1_000_000, + self.bytes + ); + } + } +} diff --git a/tests/carrier_git.rs b/tests/carrier_git.rs index e67b12a..d4c7d53 100644 --- a/tests/carrier_git.rs +++ b/tests/carrier_git.rs @@ -238,6 +238,14 @@ fn carrier_git_supports_native_push_and_clone_without_plaintext() { false, ); git(&source, &["push", "private", "main"], true); + assert_eq!( + git( + &carrier, + &["show", "refs/heads/git-remote-e2ee:.gitattributes"], + false, + ), + "e2ee/** -delta" + ); let mut clone_command = Command::new("git"); clone_command @@ -476,9 +484,10 @@ fn carrier_cache_fetches_only_new_objects() { drop(cached); let remote_after = reachable_object_count(&carrier); assert!(remote_after > remote_before); + let after_push = reachable_object_count(&cache); + assert_eq!(after_push, remote_after); let updated = GitStorage::open(carrier.to_str().unwrap()).unwrap(); - let after_push = reachable_object_count(&cache); assert_eq!(after_push - before_push, remote_after - remote_before); assert_eq!(after_push, remote_after); drop(updated); diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 800043e..894d179 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -427,6 +427,131 @@ fn missing_verified_frontier_tip_fails_closed() { assert!(error.contains("verified ref refs/heads/main is missing locally")); } +#[test] +fn local_publication_advances_verified_frontier_before_observation() { + const CHILD_ENV: &str = "E2EE_TEST_FRONTIER_TRACE_CHILD"; + if std::env::var_os(CHILD_ENV).is_none() { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "local_publication_advances_verified_frontier_before_observation", + "--nocapture", + ]) + .env(CHILD_ENV, "1") + .env("GIT_REMOTE_E2EE_TRACE", "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "frontier trace child failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let trace = String::from_utf8_lossy(&output.stderr); + let counters: Vec<_> = trace + .lines() + .filter(|line| line.contains("count=git_ref_connectivity_walk_objects")) + .collect(); + assert_eq!(counters.len(), 1, "expected one connectivity walk: {trace}"); + assert!( + counters[0].ends_with("value=0"), + "the observation re-walked published history: {}", + counters[0] + ); + return; + } + + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let remote = temporary.path().join("remote"); + initialize_git(&source); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote), key); + repository.initialize().unwrap(); + + let first = commit(&source, "first\n", "first"); + repository + .push_update_for_remote( + &source, + "origin", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + assert_eq!( + client_state(&source, "origin")["verified_refs"]["refs/heads/main"], + first + ); + + let observed = repository.observe_manifest(&source, "origin").unwrap(); + assert_eq!(observed.refs.get("refs/heads/main"), Some(&first)); + + let second = commit(&source, "second\n", "second"); + repository + .push_update_for_remote( + &source, + "origin", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + assert_eq!( + client_state(&source, "origin")["verified_refs"]["refs/heads/main"], + second + ); +} + +#[test] +fn shallow_publication_does_not_advance_verified_frontier() { + let temporary = tempfile::tempdir().unwrap(); + let full_source = temporary.path().join("full-source"); + let shallow_source = temporary.path().join("shallow-source"); + let remote = temporary.path().join("remote"); + initialize_git(&full_source); + commit(&full_source, "first\n", "first"); + commit(&full_source, "second\n", "second"); + + let source_url = format!("file://{}", full_source.display()); + let clone = Command::new("git") + .args(["clone", "--quiet", "--depth=1", "--no-tags"]) + .arg(&source_url) + .arg(&shallow_source) + .output() + .unwrap(); + assert!( + clone.status.success(), + "git clone: {}", + String::from_utf8_lossy(&clone.stderr) + ); + assert_eq!( + git(&shallow_source, &["rev-parse", "--is-shallow-repository"]), + "true" + ); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote), key); + repository.initialize().unwrap(); + repository + .push_update_for_remote( + &shallow_source, + "origin", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + + assert!( + client_state(&shallow_source, "origin")["verified_refs"] + .as_object() + .unwrap() + .is_empty() + ); +} + fn read_stored_object(root: &Path, kind: &str, id: &str) -> Vec { fs::read(root.join(kind).join(&id[..2]).join(id)).unwrap() }