From 6fd1ff711d043a0427f59ae5f60abd287b0beb10 Mon Sep 17 00:00:00 2001 From: hmirin Date: Wed, 14 Oct 2026 14:27:43 +0900 Subject: [PATCH] Verify fetched history incrementally and add comparison benchmarks - Refactor storage object validation into shared helpers. - Check fetch connectivity only from the newly received objects instead of walking the whole history on every fetch. - Add benchmark scripts comparing plain Git, git-crypt-style file filters, and git-remote-gcrypt, with results in docs/benchmarks.md. --- Cargo.lock | 92 ------- Cargo.toml | 2 +- docs/benchmarks.md | 285 ++++++++++++++++++++++ docs/spec.md | 15 ++ scripts/benchmark-comparison.sh | 384 ++++++++++++++++++++++++++++++ scripts/benchmark-file-filters.sh | 210 ++++++++++++++++ scripts/benchmark-gcrypt.sh | 227 ++++++++++++++++++ src/crypto.rs | 20 +- src/git.rs | 29 ++- src/repository.rs | 60 ++++- src/storage.rs | 136 +++++++---- tests/end_to_end.rs | 34 ++- 12 files changed, 1343 insertions(+), 151 deletions(-) create mode 100755 scripts/benchmark-comparison.sh create mode 100755 scripts/benchmark-file-filters.sh create mode 100755 scripts/benchmark-gcrypt.sh diff --git a/Cargo.lock b/Cargo.lock index 7bd9c6c..b9d9077 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,12 +93,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base64" version = "0.22.1" @@ -228,18 +222,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-common" version = "0.1.7" @@ -333,25 +315,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest", - "ff", - "generic-array", - "group", - "hkdf", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "errno" version = "0.3.14" @@ -368,16 +331,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "fiat-crypto" version = "0.2.9" @@ -402,7 +355,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -473,17 +425,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "heck" version = "0.5.0" @@ -527,7 +468,6 @@ dependencies = [ "generic-array", "hkdf", "hmac", - "p256", "rand_core 0.9.5", "sha2", "subtle", @@ -592,16 +532,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "elliptic-curve", - "primeorder", -] - [[package]] name = "pkcs8" version = "0.10.2" @@ -644,15 +574,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -764,19 +685,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "subtle", - "zeroize", -] - [[package]] name = "semver" version = "1.0.28" diff --git a/Cargo.toml b/Cargo.toml index d9bd6ba..27fc7a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ clap = { version = "4.5", features = ["derive"] } ed25519-dalek = { version = "2.1", features = ["rand_core"] } fs2 = "0.4" hex = "0.4" -hpke = { version = "0.13", features = ["x25519"] } +hpke = { version = "0.13", default-features = false, features = ["alloc", "x25519"] } hkdf = "0.12" rand = "0.8" rand_09 = { package = "rand", version = "0.9", features = ["os_rng", "std_rng"], default-features = false } diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 8eb2ca8..d7fd7f4 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -7,6 +7,33 @@ against a real Git repository using an isolated filesystem backend: scripts/benchmark-local.sh /path/to/source/repository ``` +To compare the filesystem backend with a local bare Git remote, run: + +```console +scripts/benchmark-comparison.sh /path/to/source/repository +``` + +The repository also includes two opt-in competitor harnesses. They require +separate local checkouts/builds of the named projects and do not install or +modify those tools globally: + +```console +scripts/benchmark-gcrypt.sh /path/to/source/repository /path/to/git-remote-gcrypt +scripts/benchmark-file-filters.sh /path/to/git-crypt /path/to/transcrypt +``` + +The comparison benchmark sends the exact same commit sequence to both remotes: +an initial push, a tiny change, an incompressible 10 MiB file, a one-byte +modification of that file, and 1,000 unique small files. After the initial +transfer it adds a second E2EE reader, proves +that every existing pack is byte-for-byte unchanged, and fetches the complete +history with only the new reader's private key. Later updates are also fetched +by that new reader. Each push is followed by a fetch into a returning client. +The script records wall time, peak RSS, and both logical and allocated +remote/client disk growth. Git automatic maintenance is disabled so +asynchronous commit-graph or repack work cannot move the disk measurements +after a phase completes. + The script creates a transport clone through `file://` with `--no-local`, removes its `origin`, and keeps every key, encrypted remote, and reconstructed repository in a temporary directory. It never checks out the source's files and @@ -68,3 +95,261 @@ grows with reachable history even when the encrypted delta is tiny. `phases.tsv` records phase name, wall seconds, peak RSS bytes, allocated storage bytes, and logical storage bytes. `summary.json` records aggregate environment, source, pack-count, and storage-size metadata. + +## Single-run comparison with large public repositories + +On 2026-08-16 the comparison benchmark was run once per repository on the same +Apple Silicon Mac, using Rust 1.95.0, the release profile, and crate commit +`f7ef09c`. Only each default branch was cloned, without tags. These are +deliberately single-run exploratory results, not stable thresholds or medians: + +| Repository | Source HEAD | Commits | HEAD files | Reachable data | +|---|---|---:|---:|---:| +| `godotengine/godot` | `00932449c9f3` | 85,678 | 14,162 | 867 MiB | +| `rust-lang/rust` | `67854e511de2` | 336,609 | 62,013 | 926 MiB | +| `kubernetes/kubernetes` | `a231bf3f3776` | 140,375 | 31,300 | 1.16 GiB | + +Initial transfers show that streaming encryption does not add another +whole-pack memory copy. Initial push RSS is effectively the same as Git's and +is dominated by `pack-objects`; E2EE fresh-fetch RSS is lower in all three +runs. The E2EE push does less receiver-side Git processing because the +untrusted backend stores opaque objects rather than indexing plaintext Git +objects, so its local-filesystem time is not a network-server throughput claim. + +| Repository | Initial push Git / E2EE | Push RSS Git / E2EE | Remote write Git / E2EE | Fresh fetch Git / E2EE | Fetch RSS Git / E2EE | +|---|---:|---:|---:|---:|---:| +| Godot | 30.51 / 13.78 s | 1,301 / 1,331 MiB | 890 / 877 MiB | 30.26 / 31.75 s | 1,321 / 601 MiB | +| Rust | 75.77 / 34.28 s | 2,058 / 2,050 MiB | 1,031 / 970 MiB | 73.70 / 63.19 s | 2,064 / 1,429 MiB | +| Kubernetes | 45.46 / 22.38 s | 1,770 / 1,771 MiB | 1,234 / 1,203 MiB | 46.76 / 50.21 s | 1,771 / 1,053 MiB | + +The initial reconstructed client's allocated disk was effectively equal for +Godot (904 MiB each), 1,034 MiB for Git versus 1,082 MiB for E2EE on Rust, and +1,235 MiB versus 1,266 MiB on Kubernetes. Those 0--4.6% differences come from +pack/index layout rather than an additional plaintext checkout; both clients +are non-bare Git repositories with empty working trees. + +Adding a second reader was independent of repository history size in these +runs. The policy transition generated a new key and two recipient envelopes, +but did not create or rewrite a content pack: + +| Repository | `device-add` time | Peak RSS | Remote logical growth | New-reader full fetch | Fetch peak RSS | New client disk | +|---|---:|---:|---:|---:|---:|---:| +| Godot | 0.03 s | 2.73 MiB | 3,379 bytes | 31.73 s | 577 MiB | 904 MiB | +| Rust | 0.03 s | 2.73 MiB | 3,379 bytes | 62.75 s | 1,439 MiB | 1,082 MiB | +| Kubernetes | 0.03 s | 2.75 MiB | 3,379 bytes | 47.01 s | 1,052 MiB | 1,267 MiB | + +Before and after `device-add`, the benchmark compares a sorted list containing +every pack path and SHA-256 digest. All three repositories matched exactly, so +roughly 0.9--1.2 GiB of prior ciphertext remained byte-for-byte unchanged. The +new reader nevertheless reconstructed the complete history using only its own +private key. Its full-fetch cost is therefore similar to any other fresh fetch +and scales with content/history, while the administrative operation itself does +not. + +With two active readers, later manifests were about 304--305 bytes larger than +the corresponding single-reader run. This is the expected per-generation +recipient-envelope cost: metadata grows linearly with active readers, while +pack storage remains independent of reader count. + +### Scaling to 100 active readers + +The Godot workload was repeated with one administrator and 99 collaborators, +added through 99 sequential policy transitions. `BENCH_TOTAL_READERS=100` +enables this mode. Every pre-existing pack path and SHA-256 digest still matched +after all additions, and the 100th reader fetched the complete history and all +later updates using only its own private key. + +The result is bounded and practical at 100 readers, but it is not literally +constant. The immutable content is independent of reader count; policy and +recipient-envelope work is not: + +| Operation | 2 active readers | 100 active readers | +|---|---:|---:| +| Last `device-add` | 0.03 s, 3,379 bytes | 0.12 s, 82,827 bytes | +| New reader full fetch | 31.73 s, 577 MiB RSS | 37.45 s, 575 MiB RSS | +| Returning tiny fetch | 3.24 s, 526 MiB RSS | 3.50 s, 528 MiB RSS | +| New client allocated disk | 904 MiB | 904 MiB | +| Existing pack rewrite | 0 bytes | 0 bytes | + +Adding readers 2 through 100 took 6.42 seconds in total and retained 4.07 MiB +of logical policy/manifest metadata (4.45 MiB allocated). The 100th individual +transition was still small, but the history of adding readers one at a time is +quadratic in aggregate metadata: each immutable policy generation repeats a +larger device list and each manifest repeats a larger envelope list. This does +not multiply content packs. + +Push timing also depends on the client path. The direct diagnostic +`git-e2ee push` has no clone-local continuity pin, so it deliberately validates +the complete 100-generation chain on every push. It took 3.12--3.54 seconds in +this run. Normal Git usage goes through `git-remote-e2ee`; set +`BENCH_E2EE_PUSH_MODE=native` to benchmark that path. Its first push after 99 +unobserved membership changes spent 3.25 seconds catching its continuity pin +up once. Later steady-state pushes were: + +| Update | 2 readers, direct CLI | 100 readers, native Git | +|---|---:|---:| +| Add incompressible 10 MiB | 0.40 s | 0.65 s | +| Add 1,000 small files | 0.34 s | 0.60 s | + +At 100 readers each later manifest added about 32.5 KiB, roughly 29 KiB more +than the two-reader run. Peak push RSS stayed small (about 17--29 MiB after the +one-time catch-up), and remote growth remained the new Git pack plus that small +manifest. Thus reader count does not affect bulk encryption, pack storage, or +client repository size, while HPKE envelope creation adds about 0.25 seconds +and tens of KiB per generation at 100 readers on this machine. + +Incremental push time stayed below one second for every case. Logical remote +growth was approximately 1.4--1.6 KiB for Git versus 3.1--3.3 KiB for E2EE for +the tiny commit, 10,244.5--10,244.7 KiB versus 10,246.4--10,246.5 KiB for the +10 MiB file, and about 110.2 KiB versus 79.3 KiB for the 1,000-file commit. +Pack compression and framing differ, so the smaller side is workload-dependent; +the useful result is that growth remains proportional to new objects rather +than repository size. + +| Repository | Tiny push Git / E2EE | 10 MiB push Git / E2EE | 1,000-file push Git / E2EE | +|---|---:|---:|---:| +| Godot | 0.17 / 0.08 s | 0.60 / 0.40 s | 0.56 / 0.34 s | +| Rust | 0.37 / 0.08 s | 0.62 / 0.40 s | 0.59 / 0.36 s | +| Kubernetes | 0.25 / 0.08 s | 0.58 / 0.39 s | 0.50 / 0.34 s | + +Before verified-frontier connectivity was implemented, the clear bottleneck +was returning-client fetch. E2EE performed a full connectivity walk after +importing even a tiny encrypted delta, so the difference grew with history +size: + +| Repository | Tiny returning fetch Git / E2EE | E2EE peak RSS | +|---|---:|---:| +| Godot | 0.08 / 3.24 s | 526 MiB | +| Rust | 0.35 / 19.65 s | 1,436 MiB | +| Kubernetes | 0.26 / 6.62 s | 1,053 MiB | + +Adding 10 MiB or 1,000 files barely changed those old E2EE returning-fetch +times. This identified history traversal, rather than AEAD throughput or delta +size, as the dominant incremental-read cost. + +The implementation now persists the exact ref tips that passed connectivity +verification. A returning fetch requires those frontier tips to remain local +and asks `rev-list` to inspect only objects newly reachable beyond them. A +missing or legacy frontier still triggers a full walk, and the signed-ref +without-pack attack test now exercises and passes through the incremental path. + +Godot was rerun once with the optimization and the additional one-byte-change +scenario. Initial/fresh operations remained unchanged within single-run noise: +initial E2EE push was 14.32 seconds, fresh fetch was 31.86 seconds, and a new +reader's full fetch was 31.70 seconds. Returning fetch changed substantially: + +| Godot returning update | Plain Git | E2EE before | E2EE verified frontier | +|---|---:|---:|---:| +| Tiny commit | 0.12 s | 3.24 s | 0.09 s | +| Add incompressible 10 MiB | 0.58 s | about 3.2 s | 0.23 s | +| Change one byte in 10 MiB | 0.37 s | not measured | 0.23 s | +| Add 1,000 small files | 0.08 s | about 3.2 s | 0.10 s | + +Peak E2EE RSS for those returning fetches was 39--40 MiB, down from 526 MiB +for the old Godot tiny-fetch path. The reconstructed clients still resolved to +the expected final ref and passed `git fsck --full`; the encrypted remote also +passed full authenticated-object verification. Rust and Kubernetes have not +yet been rerun with this optimization, so their old values above remain useful +as the before baseline rather than a claim about current performance. + +All six reconstructed clients (Git and E2EE for each repository) resolved to +the expected synthetic final commit and passed `git fsck --full`; every E2EE +remote also passed full authenticated-object verification. Rust's upstream +history produced two pre-existing `badFilemode` warnings in both reconstructed +clients, without verification failure. + +Remote logical growth is a useful approximation for local-backend write volume, +not a direct network-byte measurement. Filesystem allocation is also recorded +in the raw TSV. Results can be affected by OS page cache and local filesystem +behavior; repeat at least three times and report medians before making release +claims. + +An earlier pass exposed raw Git's detached post-receive automatic maintenance: +remote metadata could grow between two otherwise read-only measurement phases. +The results above are the clean rerun after disabling `receive.autogc` and +automatic client maintenance. The tables report operation time and immediate +per-push logical delta so phase boundaries remain explicit. + +## Comparison with the tools in the feature table + +The tools in the README do not all encrypt at the same layer, so one combined +ranking would be misleading. The following are two separate, single-run local +workloads on the same Apple Silicon Mac on 2026-08-16. They are exploratory +measurements, not release thresholds. OS page cache and process startup affect +the results; repeat the harnesses and report medians for stronger claims. + +Tool revisions were `git-remote-gcrypt` `a5ff704d071f`, `git-crypt` +`8c7a90ff38fc`, `transcrypt` `1b59c8e505c0`, and `git-remote-e2ee` +`f7ef09c` plus the uncommitted benchmark harnesses described here. + +### Whole encrypted remote: Godot + +This is the direct comparison. Each whole-remote tool received the same Godot +default-branch history (85,678 commits, 14,162 HEAD files, 867 MiB reachable +data), followed by the same synthetic commits. `git-remote-gcrypt` used its +local-filesystem backend, which its own documentation identifies as an +efficient backend. This does **not** represent its arbitrary Git or SFTP +transport, for which its documentation warns that complete history may be +uploaded on each push. + +| Metric | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial push | 30.51 s | 12.26 s | 13.78 s | +| Initial push peak RSS | 1,301 MiB | 1,006 MiB | 1,331 MiB | +| Initial remote write | 890 MiB | 877 MiB | 877 MiB | +| Fresh fetch | 30.26 s | 30.78 s | 31.75 s | +| Fresh fetch peak RSS | 1,321 MiB | 581 MiB | 601 MiB | +| Tiny push | 0.17 s | 0.54 s | 0.08 s | +| Incompressible 10 MiB push | 0.60 s | 0.66 s | 0.40 s | +| 1,000-small-file push | 0.56 s | 0.69 s | 0.34 s | + +The result is narrower than a feature-only comparison might suggest: +gcrypt's local backend is essentially tied with E2EE for initial transfer. +E2EE is faster for these incremental pushes. Before verified-frontier +connectivity, gcrypt's returning fetches were also much faster. After the +optimization, the same update classes took 0.50/0.64/0.47 seconds in gcrypt +versus 0.09/0.23/0.10 seconds in E2EE for tiny/10 MiB/1,000-file fetches. +These are separate single runs and not stable percentage claims, but the old +full-history connectivity bottleneck is no longer present on this workload. + +The gcrypt remote grew by about 1.7 KiB, 10.01 MiB, and 67.2 KiB for the three +updates. E2EE grew by about 3.2 KiB, 10.01 MiB, and 79.3 KiB. Both local +backends therefore stored incremental data rather than rewriting the 877 MiB +history. Backend choice matters: these gcrypt numbers must not be generalized +to its Git-hosted transport. + +### Selected encrypted files: filter microbenchmark + +`git-crypt` and `transcrypt` leave the repository itself visible and encrypt +only matching blobs during `git add`. Running them on every object in the +Godot history would measure a different product contract, so the harness uses +a fresh ordinary Git repository with one selected 10 MiB random file and +1,000 selected 1 KiB random files. It measures filter staging plus a push to a +local bare Git remote. A fresh clone is then unlocked and byte-compared with +the plaintext inputs. The committed blobs are also checked to differ from the +plaintext blob ID, and both clones pass `git fsck --full`. + +| Operation | `git-crypt` | `transcrypt` | +|---|---:|---:| +| Add 10 MiB: stage + push | 1.14 s | 1.76 s | +| Add 10 MiB: remote logical growth | 10.00 MiB | 10.52 MiB | +| Change one byte in that file: stage + push | 1.27 s | 2.00 s | +| One-byte change: remote logical growth | 10.00 MiB | 10.52 MiB | +| Add 1,000 small files: stage + push | 20.03 s | 143.94 s | +| 1,000 files: remote logical growth | 1.06 MiB | 1.12 MiB | +| Fresh clone + unlock | 29.64 s | 380.18 s | +| Peak RSS across measured filter phases | 74.8 MiB | 92.0 MiB | + +For large individual files the filter cost is modest. File count is the more +important result: clean/smudge process startup dominates 1,000 small files, +especially for transcrypt. Filter tools also destroy plaintext similarity +before Git stores the selected blob: in this un-repacked local remote, a +one-byte plaintext change added another complete ciphertext-sized object. +That behavior is inherent to per-file ciphertext, although later Git +maintenance and filesystem compression can change allocated-disk results. + +These numbers should not be read as saying that a whole-remote helper is +universally faster. The filter tools preserve host-side diffs and normal Git +features for everything not selected, use little memory, and avoid encrypting +unselected history. Conversely, their shared-secret and metadata-hiding models +are not substitutes for E2EE's per-device whole-repository design. diff --git a/docs/spec.md b/docs/spec.md index c24b1f1..d2ce557 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -319,6 +319,21 @@ refs, pinning the new head, or publishing a successor. A signed ref advance whose required pack delta is absent is invalid even when every cryptographic check succeeds. +A client MAY satisfy that requirement incrementally. After a successful full +or incremental connectivity check, it records the exact checked ref tips as a +verified frontier. On a later fetch it first requires every frontier tip to +still exist locally, then walks the current tips while excluding objects +reachable from the verified frontier. Content-addressed Git objects reachable +from that frontier were already checked and cannot be changed in place, so an +inductive check of only the newly reachable range establishes completeness of +the new state. The frontier advances only after pack authentication, import, +and the incremental connectivity check all succeed. + +A missing frontier, including a client state written by an older version, +requires a full connectivity walk. The incremental rule assumes the trusted +client's existing object database has not been corrupted outside this +protocol; explicit full verification remains available to check local storage. + Force pushes append a normal pack delta and new ref state. Older packs remain in historical manifests and may become unreachable in the inner Git graph. diff --git a/scripts/benchmark-comparison.sh b/scripts/benchmark-comparison.sh new file mode 100755 index 0000000..956b157 --- /dev/null +++ b/scripts/benchmark-comparison.sh @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: benchmark-comparison.sh SOURCE_GIT_REPOSITORY [RESULTS_DIRECTORY] + +Compares a local bare Git remote with the git-remote-e2ee filesystem backend. +The source repository is cloned into an isolated temporary directory and is +never modified. Set BENCH_TOTAL_READERS to an integer of at least 2 to exercise +recipient scaling (default: 2). Set BENCH_KEEP_WORK=1 to retain the temporary +directory. Set BENCH_E2EE_PUSH_MODE=native to measure pushes through the Git +remote helper instead of the direct diagnostic CLI (default: direct). +EOF +} + +if [[ $# -lt 1 || $# -gt 2 ]]; then + usage + exit 2 +fi + +source_repo=$1 +total_readers=${BENCH_TOTAL_READERS:-2} +e2ee_push_mode=${BENCH_E2EE_PUSH_MODE:-direct} +if [[ ! $total_readers =~ ^[0-9]+$ ]] || (( total_readers < 2 )); then + echo "BENCH_TOTAL_READERS must be an integer of at least 2" >&2 + exit 2 +fi +if [[ $e2ee_push_mode != direct && $e2ee_push_mode != native ]]; then + echo "BENCH_E2EE_PUSH_MODE must be direct or native" >&2 + exit 2 +fi +project_root=$(cd "$(dirname "$0")/.." && pwd -P) +results_dir=${2:-$project_root/bench-results/comparison-$(date -u +%Y%m%dT%H%M%SZ)} +source_repo=$(cd "$source_repo" && pwd -P) +results_dir=$(mkdir -p "$results_dir" && cd "$results_dir" && pwd -P) + +git -C "$source_repo" rev-parse --verify HEAD >/dev/null +source_ref=$(git -C "$source_repo" symbolic-ref --quiet HEAD) || { + echo "source repository must have a symbolic branch checked out" >&2 + exit 2 +} +source_branch=${source_ref#refs/heads/} +source_head=$(git -C "$source_repo" rev-parse HEAD) +source_git_dir=$(git -C "$source_repo" rev-parse --absolute-git-dir) +source_git_kib=$(du -sk "$source_git_dir" | awk '{print $1}') +available_kib=$(df -Pk "${TMPDIR:-/tmp}" | awk 'NR == 2 {print $4}') +required_kib=$((source_git_kib * 6)) +if (( available_kib < required_kib )); then + echo "insufficient temporary disk: need about $((required_kib / 1024 / 1024)) GiB" >&2 + exit 1 +fi + +case $(uname -s) in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform for RSS measurement: $(uname -s)" >&2; exit 2 ;; +esac +[[ -x /usr/bin/time ]] || { + echo "/usr/bin/time is required" >&2 + exit 1 +} + +if [[ ${BENCH_SKIP_BUILD:-0} != 1 ]]; then + cargo build --release --bins --manifest-path "$project_root/Cargo.toml" +fi +git_e2ee="$project_root/target/release/git-e2ee" +[[ -x $git_e2ee ]] || { + echo "release binary not found: $git_e2ee" >&2 + exit 1 +} + +work_parent=${TMPDIR:-/tmp} +work=$(mktemp -d "$work_parent/git-remote-e2ee-compare.XXXXXX") +cleanup() { + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + return + fi + case "$work" in + "$work_parent"/git-remote-e2ee-compare.*) rm -rf -- "$work" ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +tree_allocated_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + else + du -sk "$path" | awk '{print $1 * 1024}' + fi +} + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {print sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {print sum + 0}' + fi +} + +object_snapshot() { + local root=$1 + local object digest + find "$root/objects" -type f -print | LC_ALL=C sort | while IFS= read -r object; do + if [[ $time_style == bsd ]]; then + digest=$(shasum -a 256 "$object" | awk '{print $1}') + else + digest=$(sha256sum "$object" | awk '{print $1}') + fi + printf '%s\t%s\n' "${object#"$root"/}" "$digest" + done +} + +parse_elapsed_gnu() { + awk -F': ' '/Elapsed \(wall clock\)/ { + n=split($2, a, ":"); + if (n == 3) print a[1] * 3600 + a[2] * 60 + a[3]; + else print a[1] * 60 + a[2]; + }' "$1" +} + +phases_tsv="$results_dir/phases.tsv" +printf 'scenario\ttransport\toperation\twall_seconds\tpeak_rss_bytes\tremote_allocated_bytes\tremote_allocated_delta_bytes\tremote_logical_bytes\tremote_logical_delta_bytes\tclient_allocated_bytes\tclient_allocated_delta_bytes\n' >"$phases_tsv" + +run_phase() { + local scenario=$1 + local transport=$2 + local operation=$3 + local remote_path=$4 + local client_path=$5 + shift 5 + + local label="${scenario}-${transport}-${operation}" + local timing="$work/$label.time" + local stdout="$work/$label.stdout" + local stderr="$work/$label.stderr" + local remote_allocated_before remote_logical_before client_allocated_before + local wall rss + remote_allocated_before=$(tree_allocated_bytes "$remote_path") + remote_logical_before=$(tree_logical_bytes "$remote_path") + client_allocated_before=$(tree_allocated_bytes "$client_path") + + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,80p' "$stderr" >&2 + return 1 + fi + wall=$(awk '/ real / {print $1; exit}' "$timing") + rss=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,80p' "$stderr" >&2 + return 1 + fi + wall=$(parse_elapsed_gnu "$timing") + rss_kib=$(awk -F': ' '/Maximum resident set size/ {print $2; exit}' "$timing") + rss=$((rss_kib * 1024)) + fi + + local remote_allocated_after remote_logical_after client_allocated_after + remote_allocated_after=$(tree_allocated_bytes "$remote_path") + remote_logical_after=$(tree_logical_bytes "$remote_path") + client_allocated_after=$(tree_allocated_bytes "$client_path") + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$scenario" "$transport" "$operation" "$wall" "$rss" \ + "$remote_allocated_after" "$((remote_allocated_after - remote_allocated_before))" \ + "$remote_logical_after" "$((remote_logical_after - remote_logical_before))" \ + "$client_allocated_after" "$((client_allocated_after - client_allocated_before))" \ + >>"$phases_tsv" +} + +make_commit() { + local scenario=$1 + local old_head tree index_file new_head + old_head=$(git -C "$work/source" rev-parse "$source_ref") + tree=$(git -C "$work/source" rev-parse "$old_head^{tree}") + index_file="$work/${scenario}.index" + GIT_INDEX_FILE="$index_file" git -C "$work/source" read-tree "$tree" + + case $scenario in + tiny_change) + blob=$(printf 'git-remote-e2ee comparison benchmark tiny change\n' | + git -C "$work/source" hash-object -w --stdin) + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.git-e2ee-benchmark/tiny-change.txt" + ;; + add_10mib) + dd if=/dev/urandom of="$work/random-10mib.bin" \ + bs=1048576 count=10 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.git-e2ee-benchmark/random-10mib.bin" + ;; + modify_10mib_one_byte) + cp "$work/random-10mib.bin" "$work/random-10mib-mutated.bin" + old_byte=$(dd if="$work/random-10mib-mutated.bin" bs=1 skip=5242880 count=1 \ + 2>/dev/null | od -An -tu1 | tr -d ' ') + new_byte=$(((old_byte + 1) % 256)) + replacement=$(printf '%03o' "$new_byte") + printf '%b' "\\$replacement" | dd of="$work/random-10mib-mutated.bin" \ + bs=1 seek=5242880 conv=notrunc 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib-mutated.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.git-e2ee-benchmark/random-10mib.bin" + ;; + add_1000_small) + index_info="$work/${scenario}.index-info" + : >"$index_info" + for n in $(seq 1 1000); do + content=$(printf 'small benchmark file %04d\n%s\n' "$n" \ + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef') + blob=$(printf '%s' "$content" | git -C "$work/source" hash-object -w --stdin) + printf '100644 %s\t.git-e2ee-benchmark/small/file-%04d.txt\n' "$blob" "$n" \ + >>"$index_info" + done + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --index-info <"$index_info" + ;; + *) echo "unknown scenario: $scenario" >&2; return 2 ;; + esac + + tree=$(GIT_INDEX_FILE="$index_file" git -C "$work/source" write-tree) + new_head=$(printf 'benchmark: %s\n' "$scenario" | + git -C "$work/source" -c user.name=Benchmark \ + -c user.email=benchmark@example.invalid commit-tree "$tree" -p "$old_head") + git -C "$work/source" update-ref "$source_ref" "$new_head" "$old_head" +} + +echo "creating isolated source clone" >&2 +git clone --quiet --no-local --no-checkout --single-branch --branch "$source_branch" \ + "file://$source_repo" "$work/source" +git -C "$work/source" remote remove origin +git -C "$work/source" config gc.auto 0 +git -C "$work/source" config maintenance.auto false + +git init -q --bare "$work/raw.git" +git init -q "$work/raw-client" +git -C "$work/raw.git" config receive.autogc false +git -C "$work/raw.git" config maintenance.auto false +git -C "$work/raw-client" config gc.auto 0 +git -C "$work/raw-client" config maintenance.auto false +mkdir -p "$work/e2ee-storage" +git init -q "$work/e2ee-client" +git init -q "$work/e2ee-new-client" +git -C "$work/e2ee-client" config gc.auto 0 +git -C "$work/e2ee-client" config maintenance.auto false +git -C "$work/e2ee-new-client" config gc.auto 0 +git -C "$work/e2ee-new-client" config maintenance.auto false +"$git_e2ee" keygen --output "$work/repository.key.json" >/dev/null +"$git_e2ee" init --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" >/dev/null +if [[ $e2ee_push_mode == native ]]; then + git -C "$work/source" remote add private "e2ee::$work/e2ee-storage" + git -C "$work/source" config remote.private.e2ee-key "$work/repository.key.json" + e2ee_push_command=( + env "PATH=$project_root/target/release:$PATH" + git -C "$work/source" push --quiet private "$source_ref:$source_ref" + ) +else + e2ee_push_command=( + "$git_e2ee" push --storage "$work/e2ee-storage" + --key "$work/repository.key.json" --repo "$work/source" --ref "$source_ref" + ) +fi + +run_phase initial git push "$work/raw.git" "$work/source" \ + git -C "$work/source" push --quiet "$work/raw.git" "$source_ref:$source_ref" +run_phase initial e2ee push "$work/e2ee-storage" "$work/source" \ + "${e2ee_push_command[@]}" +run_phase initial git fetch "$work/raw.git" "$work/raw-client" \ + git -C "$work/raw-client" fetch --quiet "$work/raw.git" \ + "$source_ref:refs/remotes/benchmark/$source_branch" +run_phase initial e2ee fetch "$work/e2ee-storage" "$work/e2ee-client" \ + "$git_e2ee" fetch --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" --repo "$work/e2ee-client" \ + --remote-name benchmark + +repository_root=$(sed -n 's/^[[:space:]]*"repository_root": "\([^"]*\)",$/\1/p' \ + "$work/repository.key.json") +[[ -n $repository_root ]] +"$git_e2ee" keygen --repository-root "$repository_root" \ + --output "$work/new-user.key.json" >/dev/null +"$git_e2ee" device-export --key "$work/new-user.key.json" \ + --output "$work/new-user.public.json" >/dev/null +object_snapshot "$work/e2ee-storage" >"$work/objects-before-device-add.sha256" +run_phase add_user e2ee device-add "$work/e2ee-storage" "$work/e2ee-new-client" \ + "$git_e2ee" device-add --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" --device "$work/new-user.public.json" +active_reader_key="$work/new-user.key.json" +if (( total_readers > 2 )); then + for reader_number in $(seq 3 "$total_readers"); do + reader_label=$(printf '%03d' "$reader_number") + reader_key="$work/reader-$reader_label.key.json" + reader_public="$work/reader-$reader_label.public.json" + "$git_e2ee" keygen --repository-root "$repository_root" \ + --output "$reader_key" >/dev/null + "$git_e2ee" device-export --key "$reader_key" \ + --output "$reader_public" >/dev/null + run_phase "add_reader_$reader_label" e2ee device-add \ + "$work/e2ee-storage" "$work/e2ee-new-client" \ + "$git_e2ee" device-add --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" --device "$reader_public" + active_reader_key=$reader_key + done +fi +object_snapshot "$work/e2ee-storage" >"$work/objects-after-device-add.sha256" +cmp "$work/objects-before-device-add.sha256" "$work/objects-after-device-add.sha256" +run_phase add_user e2ee fetch-new-user "$work/e2ee-storage" \ + "$work/e2ee-new-client" \ + "$git_e2ee" fetch --storage "$work/e2ee-storage" \ + --key "$active_reader_key" --repo "$work/e2ee-new-client" \ + --remote-name benchmark +initial_head=$(git -C "$work/source" rev-parse "$source_ref") +new_user_head=$(git -C "$work/e2ee-new-client" \ + rev-parse "refs/remotes/benchmark/$source_branch") +[[ $new_user_head == "$initial_head" ]] + +for scenario in tiny_change add_10mib modify_10mib_one_byte add_1000_small; do + make_commit "$scenario" + run_phase "$scenario" git push "$work/raw.git" "$work/source" \ + git -C "$work/source" push --quiet "$work/raw.git" "$source_ref:$source_ref" + run_phase "$scenario" e2ee push "$work/e2ee-storage" "$work/source" \ + "${e2ee_push_command[@]}" + run_phase "$scenario" git fetch "$work/raw.git" "$work/raw-client" \ + git -C "$work/raw-client" fetch --quiet "$work/raw.git" \ + "$source_ref:refs/remotes/benchmark/$source_branch" + run_phase "$scenario" e2ee fetch "$work/e2ee-storage" "$work/e2ee-new-client" \ + "$git_e2ee" fetch --storage "$work/e2ee-storage" \ + --key "$active_reader_key" --repo "$work/e2ee-new-client" \ + --remote-name benchmark +done + +expected=$(git -C "$work/source" rev-parse "$source_ref") +raw_received=$(git -C "$work/raw-client" rev-parse "refs/remotes/benchmark/$source_branch") +e2ee_received=$(git -C "$work/e2ee-new-client" \ + rev-parse "refs/remotes/benchmark/$source_branch") +[[ $raw_received == "$expected" ]] +[[ $e2ee_received == "$expected" ]] +git -C "$work/raw-client" fsck --full --no-dangling >/dev/null +git -C "$work/e2ee-new-client" fsck --full --no-dangling >/dev/null +"$git_e2ee" verify --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" >/dev/null + +source_commits=$(git -C "$source_repo" rev-list --count HEAD) +source_files=$(git -C "$source_repo" ls-tree -r --name-only HEAD | wc -l | tr -d ' ') +source_reachable_bytes=$(git -C "$source_repo" rev-list --disk-usage --objects HEAD | + awk '{print $1}') +crate_sha=$(git -C "$project_root" rev-parse HEAD) +if [[ -n $(git -C "$project_root" status --porcelain=v1) ]]; then + crate_dirty=true +else + crate_dirty=false +fi + +cat >"$results_dir/summary.json" <&2 +cat "$phases_tsv" diff --git a/scripts/benchmark-file-filters.sh b/scripts/benchmark-file-filters.sh new file mode 100755 index 0000000..93094a4 --- /dev/null +++ b/scripts/benchmark-file-filters.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 GIT_CRYPT_CHECKOUT TRANSCRYPT_CHECKOUT [RESULTS_DIRECTORY]" >&2 +} + +if [[ $# -lt 2 || $# -gt 3 ]]; then + usage + exit 2 +fi + +git_crypt_checkout=$(cd "$1" && pwd -P) +transcrypt_checkout=$(cd "$2" && pwd -P) +git_crypt="$git_crypt_checkout/git-crypt" +transcrypt="$transcrypt_checkout/transcrypt" +[[ -x $git_crypt && -x $transcrypt ]] + +results_dir=${3:-$(pwd -P)/bench-results/file-filters-$(date -u +%Y%m%dT%H%M%SZ)} +results_dir=$(mkdir -p "$results_dir" && cd "$results_dir" && pwd -P) + +case $(uname -s) in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform: $(uname -s)" >&2; exit 2 ;; +esac + +work_parent=${BENCH_WORK_PARENT:-/tmp} +work=$(mktemp -d "$work_parent/git-remote-e2ee-filter-bench.XXXXXX") +cleanup() { + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + return + fi + case "$work" in + "$work_parent"/git-remote-e2ee-filter-bench.*) rm -rf -- "$work" ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +tree_allocated_bytes() { + local path=$1 + if [[ -e $path ]]; then du -sk "$path" | awk '{print $1 * 1024}'; else echo 0; fi +} + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {print sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {print sum + 0}' + fi +} + +parse_elapsed_gnu() { + awk -F': ' '/Elapsed \(wall clock\)/ { + n=split($2, a, ":"); + if (n == 3) print a[1] * 3600 + a[2] * 60 + a[3]; + else print a[1] * 60 + a[2]; + }' "$1" +} + +phases_tsv="$results_dir/phases.tsv" +printf 'tool\tscenario\toperation\twall_seconds\tpeak_rss_bytes\tremote_allocated_bytes\tremote_allocated_delta_bytes\tremote_logical_bytes\tremote_logical_delta_bytes\tclient_allocated_bytes\tclient_allocated_delta_bytes\n' >"$phases_tsv" + +run_phase() { + local tool=$1 scenario=$2 operation=$3 remote_path=$4 client_path=$5 + shift 5 + local label="${tool}-${scenario}-${operation}" + local timing="$work/$label.time" stdout="$work/$label.stdout" stderr="$work/$label.stderr" + local ra_before rl_before ca_before wall rss + ra_before=$(tree_allocated_bytes "$remote_path") + rl_before=$(tree_logical_bytes "$remote_path") + ca_before=$(tree_allocated_bytes "$client_path") + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(awk '/ real / {print $1; exit}' "$timing") + rss=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(parse_elapsed_gnu "$timing") + rss_kib=$(awk -F': ' '/Maximum resident set size/ {print $2; exit}' "$timing") + rss=$((rss_kib * 1024)) + fi + local ra_after rl_after ca_after + ra_after=$(tree_allocated_bytes "$remote_path") + rl_after=$(tree_logical_bytes "$remote_path") + ca_after=$(tree_allocated_bytes "$client_path") + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$tool" "$scenario" "$operation" "$wall" "$rss" "$ra_after" \ + "$((ra_after - ra_before))" "$rl_after" "$((rl_after - rl_before))" \ + "$ca_after" "$((ca_after - ca_before))" >>"$phases_tsv" +} + +make_payloads() { + mkdir -p "$work/payloads/small" + dd if=/dev/urandom of="$work/payloads/random-10mib.bin" \ + bs=1048576 count=10 2>/dev/null + cp "$work/payloads/random-10mib.bin" "$work/payloads/random-10mib-mutated.bin" + printf '\001' | dd of="$work/payloads/random-10mib-mutated.bin" \ + bs=1 seek=5242880 conv=notrunc 2>/dev/null + for n in $(seq 1 1000); do + dd if=/dev/urandom of="$work/payloads/small/file-$(printf '%04d' "$n").bin" \ + bs=1024 count=1 2>/dev/null + done +} + +commit_repo() { + local repo=$1 message=$2 + git -C "$repo" -c user.name=Benchmark -c user.email=benchmark@example.invalid \ + commit -q -m "$message" +} + +benchmark_tool() { + local tool=$1 repo="$work/$1-source" remote="$work/$1-remote.git" + local client="$work/$1-client" secret_pattern='secret/**' + local password='benchmark-only-not-a-user-secret' + + git init -q -b main "$repo" + git init -q --bare "$remote" + git -C "$remote" symbolic-ref HEAD refs/heads/main + git -C "$repo" config gc.auto 0 + git -C "$repo" config maintenance.auto false + git -C "$repo" remote add origin "$remote" + mkdir -p "$repo/secret" + + if [[ $tool == git-crypt ]]; then + (cd "$repo" && "$git_crypt" init) + printf '%s filter=git-crypt diff=git-crypt\n' "$secret_pattern" >"$repo/.gitattributes" + (cd "$repo" && "$git_crypt" export-key "$work/git-crypt.key") + else + git -C "$repo" -c advice.detachedHead=false status --short >/dev/null + (cd "$repo" && "$transcrypt" -c aes-256-cbc -p "$password" -y >/dev/null) + (cd "$repo" && "$transcrypt" --add="$secret_pattern" >/dev/null) + fi + git -C "$repo" add .gitattributes + commit_repo "$repo" 'configure encrypted file filter' + git -C "$repo" push -q -u origin main + + cp "$work/payloads/random-10mib.bin" "$repo/secret/random-10mib.bin" + run_phase "$tool" add_10mib stage "$remote" "$repo" \ + git -C "$repo" add secret/random-10mib.bin + commit_repo "$repo" 'add encrypted 10 MiB file' + run_phase "$tool" add_10mib push "$remote" "$repo" \ + git -C "$repo" push -q origin main + + cp "$work/payloads/random-10mib-mutated.bin" "$repo/secret/random-10mib.bin" + run_phase "$tool" modify_10mib_one_byte stage "$remote" "$repo" \ + git -C "$repo" add secret/random-10mib.bin + commit_repo "$repo" 'modify one byte in encrypted 10 MiB file' + run_phase "$tool" modify_10mib_one_byte push "$remote" "$repo" \ + git -C "$repo" push -q origin main + + mkdir -p "$repo/secret/small" + cp -R "$work/payloads/small/." "$repo/secret/small/" + run_phase "$tool" add_1000_small stage "$remote" "$repo" \ + git -C "$repo" add secret/small + commit_repo "$repo" 'add 1000 encrypted small files' + run_phase "$tool" add_1000_small push "$remote" "$repo" \ + git -C "$repo" push -q origin main + + plaintext_hash=$(git -C "$repo" hash-object --no-filters secret/random-10mib.bin) + ciphertext_hash=$(git -C "$repo" rev-parse HEAD:secret/random-10mib.bin) + [[ $plaintext_hash != "$ciphertext_hash" ]] + + run_phase "$tool" fresh_reader clone "$remote" "$client" \ + git clone -q "$remote" "$client" + git -C "$client" config gc.auto 0 + git -C "$client" config maintenance.auto false + if [[ $tool == git-crypt ]]; then + run_phase "$tool" fresh_reader unlock "$remote" "$client" \ + bash -c 'cd "$1" && "$2" unlock "$3"' \ + _ "$client" "$git_crypt" "$work/git-crypt.key" + else + run_phase "$tool" fresh_reader unlock "$remote" "$client" \ + bash -c 'cd "$1" && "$2" -c aes-256-cbc -p "$3" -y >/dev/null' \ + _ "$client" "$transcrypt" "$password" + fi + cmp "$work/payloads/random-10mib-mutated.bin" "$client/secret/random-10mib.bin" + git -C "$client" fsck --full --no-dangling >/dev/null +} + +make_payloads +benchmark_tool git-crypt +benchmark_tool transcrypt + +cat >"$results_dir/summary.json" <&2 +cat "$phases_tsv" diff --git a/scripts/benchmark-gcrypt.sh b/scripts/benchmark-gcrypt.sh new file mode 100755 index 0000000..ee07ed4 --- /dev/null +++ b/scripts/benchmark-gcrypt.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 SOURCE_GIT_REPOSITORY GCRYPT_CHECKOUT [RESULTS_DIRECTORY]" >&2 +} + +if [[ $# -lt 2 || $# -gt 3 ]]; then + usage + exit 2 +fi + +source_repo=$(cd "$1" && pwd -P) +gcrypt_checkout=$(cd "$2" && pwd -P) +results_dir=${3:-$(pwd -P)/bench-results/gcrypt-$(date -u +%Y%m%dT%H%M%SZ)} +results_dir=$(mkdir -p "$results_dir" && cd "$results_dir" && pwd -P) +gcrypt_helper="$gcrypt_checkout/git-remote-gcrypt" +[[ -x $gcrypt_helper ]] + +source_ref=$(git -C "$source_repo" symbolic-ref --quiet HEAD) || { + echo "source repository must have a symbolic branch checked out" >&2 + exit 2 +} +source_branch=${source_ref#refs/heads/} +source_head=$(git -C "$source_repo" rev-parse HEAD) + +case $(uname -s) in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform: $(uname -s)" >&2; exit 2 ;; +esac + +work_parent=${BENCH_WORK_PARENT:-/tmp} +work=$(mktemp -d "$work_parent/git-remote-e2ee-gcrypt-bench.XXXXXX") +cleanup() { + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + return + fi + case "$work" in + "$work_parent"/git-remote-e2ee-gcrypt-bench.*) rm -rf -- "$work" ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +tree_allocated_bytes() { + local path=$1 + if [[ -e $path ]]; then du -sk "$path" | awk '{print $1 * 1024}'; else echo 0; fi +} + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {print sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {print sum + 0}' + fi +} + +parse_elapsed_gnu() { + awk -F': ' '/Elapsed \(wall clock\)/ { + n=split($2, a, ":"); + if (n == 3) print a[1] * 3600 + a[2] * 60 + a[3]; + else print a[1] * 60 + a[2]; + }' "$1" +} + +phases_tsv="$results_dir/phases.tsv" +printf 'scenario\toperation\twall_seconds\tpeak_rss_bytes\tremote_allocated_bytes\tremote_allocated_delta_bytes\tremote_logical_bytes\tremote_logical_delta_bytes\tclient_allocated_bytes\tclient_allocated_delta_bytes\n' >"$phases_tsv" + +run_phase() { + local scenario=$1 operation=$2 remote_path=$3 client_path=$4 + shift 4 + local label="${scenario}-${operation}" + local timing="$work/$label.time" stdout="$work/$label.stdout" stderr="$work/$label.stderr" + local ra_before rl_before ca_before wall rss + ra_before=$(tree_allocated_bytes "$remote_path") + rl_before=$(tree_logical_bytes "$remote_path") + ca_before=$(tree_allocated_bytes "$client_path") + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(awk '/ real / {print $1; exit}' "$timing") + rss=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(parse_elapsed_gnu "$timing") + rss_kib=$(awk -F': ' '/Maximum resident set size/ {print $2; exit}' "$timing") + rss=$((rss_kib * 1024)) + fi + local ra_after rl_after ca_after + ra_after=$(tree_allocated_bytes "$remote_path") + rl_after=$(tree_logical_bytes "$remote_path") + ca_after=$(tree_allocated_bytes "$client_path") + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$scenario" "$operation" "$wall" "$rss" "$ra_after" \ + "$((ra_after - ra_before))" "$rl_after" "$((rl_after - rl_before))" \ + "$ca_after" "$((ca_after - ca_before))" >>"$phases_tsv" +} + +make_commit() { + local scenario=$1 old_head tree index_file new_head blob + old_head=$(git -C "$work/source" rev-parse "$source_ref") + tree=$(git -C "$work/source" rev-parse "$old_head^{tree}") + index_file="$work/${scenario}.index" + GIT_INDEX_FILE="$index_file" git -C "$work/source" read-tree "$tree" + case $scenario in + tiny_change) + blob=$(printf 'git-remote-gcrypt comparison tiny change\n' | + git -C "$work/source" hash-object -w --stdin) + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.gcrypt-benchmark/tiny-change.txt" + ;; + add_10mib) + dd if=/dev/urandom of="$work/random-10mib.bin" \ + bs=1048576 count=10 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.gcrypt-benchmark/random-10mib.bin" + ;; + modify_10mib_one_byte) + cp "$work/random-10mib.bin" "$work/random-10mib-mutated.bin" + old_byte=$(dd if="$work/random-10mib-mutated.bin" bs=1 skip=5242880 count=1 \ + 2>/dev/null | od -An -tu1 | tr -d ' ') + new_byte=$(((old_byte + 1) % 256)) + replacement=$(printf '%03o' "$new_byte") + printf '%b' "\\$replacement" | dd of="$work/random-10mib-mutated.bin" \ + bs=1 seek=5242880 conv=notrunc 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib-mutated.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.gcrypt-benchmark/random-10mib.bin" + ;; + add_1000_small) + index_info="$work/${scenario}.index-info" + : >"$index_info" + for n in $(seq 1 1000); do + blob=$(printf 'small gcrypt benchmark file %04d\n' "$n" | + git -C "$work/source" hash-object -w --stdin) + printf '100644 %s\t.gcrypt-benchmark/small/file-%04d.txt\n' "$blob" "$n" \ + >>"$index_info" + done + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --index-info <"$index_info" + ;; + *) echo "unknown scenario: $scenario" >&2; return 2 ;; + esac + tree=$(GIT_INDEX_FILE="$index_file" git -C "$work/source" write-tree) + new_head=$(printf 'benchmark: %s\n' "$scenario" | + git -C "$work/source" -c user.name=Benchmark \ + -c user.email=benchmark@example.invalid commit-tree "$tree" -p "$old_head") + git -C "$work/source" update-ref "$source_ref" "$new_head" "$old_head" +} + +mkdir -m 700 "$work/gnupg" +export GNUPGHOME="$work/gnupg" +gpg --batch --pinentry-mode loopback --passphrase '' --quick-generate-key \ + 'Benchmark ' ed25519 sign 0 >/dev/null +fingerprint=$(gpg --batch --with-colons --list-secret-keys | + awk -F: '$1 == "fpr" {print $10; exit}') +gpg --batch --pinentry-mode loopback --passphrase '' \ + --quick-add-key "$fingerprint" cv25519 encr 0 \ + >/dev/null + +echo "creating isolated source clone" >&2 +git clone --quiet --no-local --no-checkout --single-branch --branch "$source_branch" \ + "file://$source_repo" "$work/source" +git -C "$work/source" remote remove origin +git -C "$work/source" config gc.auto 0 +git -C "$work/source" config maintenance.auto false +git -C "$work/source" remote add crypt "gcrypt::$work/remote" +git -C "$work/source" config remote.crypt.gcrypt-participants "$fingerprint" +git -C "$work/source" config remote.crypt.gcrypt-signingkey "$fingerprint" +git -C "$work/source" config remote.crypt.gcrypt-require-explicit-force-push true + +git init -q "$work/client" +git -C "$work/client" config gc.auto 0 +git -C "$work/client" config maintenance.auto false +git -C "$work/client" remote add crypt "gcrypt::$work/remote" +git -C "$work/client" config remote.crypt.gcrypt-participants "$fingerprint" +git -C "$work/client" config remote.crypt.gcrypt-signingkey "$fingerprint" + +export PATH="$gcrypt_checkout:$PATH" +run_phase initial push "$work/remote" "$work/source" \ + git -C "$work/source" push --quiet --force crypt "$source_ref:$source_ref" +run_phase initial fetch "$work/remote" "$work/client" \ + git -C "$work/client" fetch --quiet crypt \ + "$source_ref:refs/remotes/crypt/$source_branch" + +for scenario in tiny_change add_10mib modify_10mib_one_byte add_1000_small; do + make_commit "$scenario" + run_phase "$scenario" push "$work/remote" "$work/source" \ + git -C "$work/source" push --quiet --force crypt "$source_ref:$source_ref" + run_phase "$scenario" fetch "$work/remote" "$work/client" \ + git -C "$work/client" fetch --quiet crypt \ + "$source_ref:refs/remotes/crypt/$source_branch" +done + +expected=$(git -C "$work/source" rev-parse "$source_ref") +received=$(git -C "$work/client" rev-parse "refs/remotes/crypt/$source_branch") +[[ $received == "$expected" ]] +git -C "$work/client" fsck --full --no-dangling >/dev/null + +cat >"$results_dir/summary.json" <&2 +cat "$phases_tsv" diff --git a/src/crypto.rs b/src/crypto.rs index ba9c5b9..2988379 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -127,13 +127,9 @@ impl KeyFile { let signing_public = self.signing_key()?.verifying_key().to_bytes(); let wrapping_private = self.wrapping_private_key()?; let wrapping_public = HpkeKem::sk_to_pk(&wrapping_private).to_bytes(); - let mut identity = Vec::with_capacity(96); - identity.extend_from_slice(b"git-remote-e2ee device id v3\0"); - identity.extend_from_slice(&signing_public); - identity.extend_from_slice(&wrapping_public); Ok(PublicDevice { repository_root: self.repository_root.clone(), - device_id: hex::encode(Sha256::digest(&identity)), + device_id: device_id_for_public_keys(&signing_public, wrapping_public.as_slice()), signing_public_key: BASE64.encode(signing_public), wrapping_public_key: BASE64.encode(wrapping_public), }) @@ -577,11 +573,7 @@ pub fn validate_public_device(device: &PublicDevice) -> Result<()> { validate_root(&device.repository_root)?; let signing = decode_public_key(&device.signing_public_key)?; let wrapping = decode_public_key(&device.wrapping_public_key)?; - let mut identity = Vec::with_capacity(96); - identity.extend_from_slice(b"git-remote-e2ee device id v3\0"); - identity.extend_from_slice(&signing); - identity.extend_from_slice(&wrapping); - if device.device_id != hex::encode(Sha256::digest(&identity)) { + if device.device_id != device_id_for_public_keys(&signing, &wrapping) { bail!("device id does not match its public keys") } Ok(()) @@ -602,6 +594,14 @@ fn repository_root_for_public_keys(signing: &[u8], wrapping: &[u8]) -> String { hex::encode(Sha256::digest(identity)) } +fn device_id_for_public_keys(signing: &[u8], wrapping: &[u8]) -> String { + let mut identity = Vec::with_capacity(32 + signing.len() + wrapping.len()); + identity.extend_from_slice(b"git-remote-e2ee device id v3\0"); + identity.extend_from_slice(signing); + identity.extend_from_slice(wrapping); + hex::encode(Sha256::digest(identity)) +} + fn validate_root(root: &str) -> Result<()> { if root.len() != 64 || !root.bytes().all(|byte| byte.is_ascii_hexdigit()) { bail!("repository root must be a 32-byte lowercase hexadecimal value") diff --git a/src/git.rs b/src/git.rs index 1204401..3046ab9 100644 --- a/src/git.rs +++ b/src/git.rs @@ -203,7 +203,11 @@ fn read_process_error(file: &mut File) -> String { message.trim().to_owned() } -pub fn ensure_refs_connected(repo: &Path, refs: &BTreeMap) -> Result<()> { +pub fn ensure_refs_connected_since( + repo: &Path, + refs: &BTreeMap, + verified_refs: &BTreeMap, +) -> Result<()> { if refs.is_empty() { return Ok(()); } @@ -219,6 +223,23 @@ pub fn ensure_refs_connected(repo: &Path, refs: &BTreeMap) -> Re } } + // A successful prior connectivity check makes every object reachable from + // these tips a trusted frontier. Git objects are immutable and + // content-addressed, so a later fetch only needs to walk objects newly + // reachable beyond that frontier. Still require the frontier tips to be + // present locally so corruption or an unexpected local prune fails closed. + for (reference, object) in verified_refs { + let expression = format!("{object}^{{commit}}"); + let output = Command::new("git") + .arg("-C") + .arg(repo) + .args(["cat-file", "-e", &expression]) + .output()?; + if !output.status.success() { + bail!("verified ref {reference} is missing locally") + } + } + let mut command = Command::new("git"); command .arg("-C") @@ -227,6 +248,12 @@ pub fn ensure_refs_connected(repo: &Path, refs: &BTreeMap) -> Re for object in refs.values() { command.arg(object); } + if !verified_refs.is_empty() { + command.arg("--not"); + for object in verified_refs.values() { + command.arg(object); + } + } let output = command.output()?; if !output.status.success() { bail!( diff --git a/src/repository.rs b/src/repository.rs index b8c20e4..df7afdc 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -35,6 +35,8 @@ struct ClientState { policy_generation: Option, #[serde(default)] repository_root: Option, + #[serde(default)] + verified_refs: BTreeMap, } #[derive(Clone)] @@ -104,7 +106,13 @@ impl EncryptedRepository { &next_key, next.clone(), )?; - write_client_state(pin_path, &HashSet::new(), &result.0, &next)?; + write_client_state( + pin_path, + &HashSet::new(), + &result.0, + &next, + &BTreeMap::new(), + )?; Ok(result) } @@ -141,7 +149,13 @@ impl EncryptedRepository { &next_key, next.clone(), )?; - write_client_state(pin_path, &HashSet::new(), &result.0, &next)?; + write_client_state( + pin_path, + &HashSet::new(), + &result.0, + &next, + &BTreeMap::new(), + )?; Ok(result) } @@ -149,7 +163,13 @@ impl EncryptedRepository { let current = self.current_state()?; let previous = read_client_state(pin_path)?; self.validate_pinned_head(¤t, &previous)?; - write_client_state(pin_path, &HashSet::new(), ¤t.id, ¤t.manifest) + write_client_state( + pin_path, + &HashSet::new(), + ¤t.id, + ¤t.manifest, + &BTreeMap::new(), + ) } pub fn list_devices(&self) -> Result> { @@ -429,8 +449,14 @@ impl EncryptedRepository { imported.insert(descriptor.id.clone()); } } - git::ensure_refs_connected(repo, ¤t.manifest.refs)?; - write_client_state(&state_path, &imported, ¤t.id, ¤t.manifest)?; + git::ensure_refs_connected_since(repo, ¤t.manifest.refs, &state.verified_refs)?; + write_client_state( + &state_path, + &imported, + ¤t.id, + ¤t.manifest, + ¤t.manifest.refs, + )?; Ok(current.manifest) } @@ -774,6 +800,7 @@ fn write_client_state( values: &HashSet, head_id: &str, manifest: &Manifest, + verified_refs: &BTreeMap, ) -> Result<()> { let mut values: Vec<_> = values.iter().cloned().collect(); values.sort(); @@ -785,6 +812,7 @@ fn write_client_state( imported_generation: Some(manifest.generation), policy_generation: Some(manifest.policy_generation), repository_root: Some(manifest.repository_root.clone()), + verified_refs: verified_refs.clone(), }; persist_client_state(path, &state) } @@ -803,6 +831,7 @@ fn write_observed_client_state( imported_generation: previous.imported_generation, policy_generation: Some(manifest.policy_generation), repository_root: Some(manifest.repository_root.clone()), + verified_refs: previous.verified_refs, }; persist_client_state(path, &state) } @@ -849,6 +878,7 @@ pub(crate) fn testing_write_client_state( imported_generation: Some(generation), policy_generation: Some(1), repository_root: Some("durability-fixture".to_owned()), + verified_refs: BTreeMap::new(), }; persist_client_state(path, &state) } @@ -863,3 +893,23 @@ fn validate_remote_name(name: &str) -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::ClientState; + + #[test] + fn client_state_without_verified_frontier_requires_a_full_check() { + let legacy = br#"{ + "format_version": 4, + "packs": ["pack-id"], + "head_id": "head-id", + "generation": 7, + "imported_generation": 7, + "policy_generation": 2, + "repository_root": "repository-root" + }"#; + let state: ClientState = serde_json::from_slice(legacy).unwrap(); + assert!(state.verified_refs.is_empty()); + } +} diff --git a/src/storage.rs b/src/storage.rs index dff9dbb..c753e49 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -24,6 +24,47 @@ pub trait ObjectStage: Write + Send { fn finish(self: Box, id: &str) -> Result<()>; } +trait ObjectStageSink: Write + Send + Sized { + fn publish(self, id: &str) -> Result<()>; +} + +struct ValidatingObjectStage { + sink: S, + hasher: Sha256, +} + +impl ValidatingObjectStage { + fn wrap(sink: S) -> Box { + Box::new(Self { + sink, + hasher: Sha256::new(), + }) + } +} + +impl Write for ValidatingObjectStage { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let written = self.sink.write(data)?; + self.hasher.update(&data[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.sink.flush() + } +} + +impl ObjectStage for ValidatingObjectStage { + fn finish(self: Box, id: &str) -> Result<()> { + validate_id(id)?; + let Self { sink, hasher } = *self; + if hex::encode(hasher.finalize()) != id { + bail!("staged object hash does not match id") + } + sink.publish(id) + } +} + pub trait Storage { fn begin_object(&self, kind: ObjectKind) -> Result>; fn open_object(&self, kind: ObjectKind, id: &str) -> Result>; @@ -108,14 +149,11 @@ struct FilesystemObjectStage { temporary: PathBuf, root: PathBuf, kind: ObjectKind, - hasher: Sha256, } impl Write for FilesystemObjectStage { fn write(&mut self, data: &[u8]) -> std::io::Result { - let written = self.file.write(data)?; - self.hasher.update(&data[..written]); - Ok(written) + self.file.write(data) } fn flush(&mut self) -> std::io::Result<()> { @@ -123,12 +161,8 @@ impl Write for FilesystemObjectStage { } } -impl ObjectStage for FilesystemObjectStage { - fn finish(mut self: Box, id: &str) -> Result<()> { - validate_id(id)?; - if hex::encode(self.hasher.clone().finalize()) != id { - bail!("staged object hash does not match id") - } +impl ObjectStageSink for FilesystemObjectStage { + fn publish(mut self, id: &str) -> Result<()> { self.file.flush()?; self.file.sync_all()?; crate::persist::durability_checkpoint(crate::persist::STAGE_AFTER_FILE_FLUSH); @@ -184,12 +218,11 @@ impl Storage for FilesystemStorage { .write(true) .create_new(true) .open(&temporary)?; - Ok(Box::new(FilesystemObjectStage { + Ok(ValidatingObjectStage::wrap(FilesystemObjectStage { file, temporary, root: self.root.clone(), kind, - hasher: Sha256::new(), })) } @@ -317,7 +350,6 @@ struct CarrierObjectStage { current: Option, chunk_index: usize, chunk_len: usize, - hasher: Sha256, } impl CarrierObjectStage { @@ -356,7 +388,6 @@ impl Write for CarrierObjectStage { .as_mut() .expect("carrier chunk opened") .write_all(&data[..take])?; - self.hasher.update(&data[..take]); self.chunk_len += take; data = &data[take..]; if self.chunk_len == CARRIER_CHUNK_SIZE { @@ -374,12 +405,8 @@ impl Write for CarrierObjectStage { } } -impl ObjectStage for CarrierObjectStage { - fn finish(mut self: Box, id: &str) -> Result<()> { - validate_id(id)?; - if hex::encode(self.hasher.clone().finalize()) != id { - bail!("staged object hash does not match id") - } +impl ObjectStageSink for CarrierObjectStage { + fn publish(mut self, id: &str) -> Result<()> { self.finish_chunk()?; if self.chunk_index == 0 { bail!("cannot store an empty carrier object") @@ -452,18 +479,16 @@ impl GitStorage { let checkout = tempfile::Builder::new() .prefix("git-remote-e2ee-carrier-") .tempdir()?; + let checkout_path = checkout + .path() + .to_str() + .context("carrier checkout path is not UTF-8")?; git_command( checkout .path() .parent() .context("carrier tempdir has no parent")?, - &[ - "clone", - "--quiet", - "--no-checkout", - remote, - checkout.path_str()?, - ], + &["clone", "--quiet", "--no-checkout", remote, checkout_path], )?; git_command(checkout.path(), &["config", "user.name", "git-remote-e2ee"])?; git_command( @@ -555,32 +580,19 @@ impl GitStorage { } } -trait TempDirPath { - fn path_str(&self) -> Result<&str>; -} - -impl TempDirPath for tempfile::TempDir { - fn path_str(&self) -> Result<&str> { - self.path() - .to_str() - .context("carrier checkout path is not UTF-8") - } -} - impl Storage for GitStorage { fn begin_object(&self, kind: ObjectKind) -> Result> { let mut random = [0_u8; 8]; OsRng.fill_bytes(&mut random); let staging = self.root().join(format!(".stage-{}", hex::encode(random))); fs::create_dir(&staging)?; - Ok(Box::new(CarrierObjectStage { + Ok(ValidatingObjectStage::wrap(CarrierObjectStage { staging, root: self.root().to_path_buf(), kind, current: None, chunk_index: 0, chunk_len: 0, - hasher: Sha256::new(), })) } @@ -695,12 +707,54 @@ fn carrier_git_command() -> Command { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Barrier}; use std::thread; use super::*; use crate::crypto::object_id; + struct PartialWriteSink { + published: Arc, + } + + impl Write for PartialWriteSink { + fn write(&mut self, data: &[u8]) -> std::io::Result { + Ok(data.len().min(3)) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl ObjectStageSink for PartialWriteSink { + fn publish(self, _id: &str) -> Result<()> { + self.published.store(true, Ordering::Relaxed); + Ok(()) + } + } + + #[test] + fn object_stage_validates_all_written_bytes_before_publishing() { + let data = b"partial writes must all contribute to the staged object hash"; + let published = Arc::new(AtomicBool::new(false)); + let mut stage = ValidatingObjectStage::wrap(PartialWriteSink { + published: Arc::clone(&published), + }); + stage.write_all(data).unwrap(); + stage.finish(&object_id(data)).unwrap(); + assert!(published.load(Ordering::Relaxed)); + + let published = Arc::new(AtomicBool::new(false)); + let mut stage = ValidatingObjectStage::wrap(PartialWriteSink { + published: Arc::clone(&published), + }); + stage.write_all(data).unwrap(); + assert!(stage.finish(&object_id(b"different bytes")).is_err()); + assert!(!published.load(Ordering::Relaxed)); + } + #[test] fn head_compare_and_swap_rejects_stale_writer() { let directory = tempfile::tempdir().unwrap(); diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 104e394..b213641 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -135,7 +135,7 @@ fn returning_client_fetches_multiple_offline_generations_from_deltas() { } #[test] -fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { +fn returning_client_rejects_signed_ref_advance_without_its_pack() { let temporary = tempfile::tempdir().unwrap(); let source = temporary.path().join("source"); let destination = temporary.path().join("destination"); @@ -149,6 +149,7 @@ fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { repository .push_ref(&source, "refs/heads/main", false) .unwrap(); + repository.fetch_into(&destination, "e2ee").unwrap(); let head = fs::read_to_string(remote.join("HEAD")) .unwrap() @@ -210,6 +211,37 @@ fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { ); } +#[test] +fn missing_verified_frontier_tip_fails_closed() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let destination = temporary.path().join("destination"); + let remote = temporary.path().join("remote"); + initialize_git(&source); + initialize_git(&destination); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote), key); + repository.initialize().unwrap(); + commit(&source, "valid\n", "valid"); + repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + repository.fetch_into(&destination, "e2ee").unwrap(); + + let state_path = destination.join(".git/git-remote-e2ee/e2ee/state.json"); + let mut state: serde_json::Value = + serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + state["verified_refs"]["refs/heads/main"] = serde_json::Value::String("1".repeat(40)); + fs::write(&state_path, serde_json::to_vec_pretty(&state).unwrap()).unwrap(); + + let error = repository + .fetch_into(&destination, "e2ee") + .unwrap_err() + .to_string(); + assert!(error.contains("verified ref refs/heads/main is missing locally")); +} + fn read_stored_object(root: &Path, kind: &str, id: &str) -> Vec { fs::read(root.join(kind).join(&id[..2]).join(id)).unwrap() }