diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..29445f6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,60 @@ +# Contributing + +Issues and pull requests are welcome. For security problems, follow +[SECURITY.md](SECURITY.md) instead of opening a public issue. + +## Ground rules + +- Keep the storage contract backend-neutral. Filesystem behavior must not leak + into encrypted repository semantics. +- Do not invent cryptographic primitives. Use established AEAD, signature, + hashing, and key-wrapping libraries. +- Treat storage as malicious for confidentiality and integrity, and document + availability, rollback, freeze, and equivocation limits precisely. +- Never commit key files or decrypted test data. Test secrets must live only in + temporary directories. +- Protocol changes must update [docs/spec.md](docs/spec.md) and, where relevant, + [docs/design.md](docs/design.md) in the same change. + +## Build and test + +```console +cargo test --all-targets +cargo clippy --all-targets -- -D warnings +cargo fmt --all -- --check +``` + +CI runs the same three commands. The ignored power-cut harness is not part of +the default run; its protocol, Windows commands, and durability limits are in +[docs/durability.md](docs/durability.md). + +The test suite covers: + +- incremental push and reconstruction into a fresh repository; +- native clone, fetch, pull, push, dry-run, refspec, and force-push behavior; +- rollback, same-generation fork, and ciphertext-tampering rejection; +- streaming-AEAD chunk boundaries, wrong keys or AAD, reordering, duplication, + truncation, trailing bytes, forged sizes, and legacy-format rejection; +- independent device add, read, and write, non-admin rejection, + genesis-substitution rejection, administrative rollback pinning, device + revocation without pack rewrites, and multi-generation offline catch-up; +- malformed predecessor-link, generation-key commitment, recipient-set, and + signed-but-incomplete Git object-graph rejection; +- two independent carrier writers racing real `git push` processes, with + exactly one winner; +- plaintext-absence checks across carrier history; +- applicable black-box scenarios independently reimplemented from Git + upstream's + [`t/t5801-remote-helpers.sh`](https://github.com/git/git/blob/master/t/t5801-remote-helpers.sh). + +A local bare Git repository is enough for deterministic carrier concurrency +tests: local-path pushes still run Git's real `receive-pack`, ref locking, and +fast-forward checks. Hosted smoke tests remain useful for authentication, +request limits, and provider-specific policy. + +## Benchmarks + +The opt-in harness in `scripts/` measures encryption, reconstruction, +verification, and incremental updates. See [docs/benchmarks.md](docs/benchmarks.md). +Benchmark outputs, repository keys, and reconstructed data must not be +committed. diff --git a/Cargo.lock b/Cargo.lock index 7bd9c6c..8aac434 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,10 +8,29 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.2", +] + +[[package]] +name = "aead-stream" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a901e5bcd15b4a9555a8f17a608d28ef92cf77bc4aa3b4a0c1a3fce1a9cc0bac" +dependencies = [ + "aead 0.6.1", +] + [[package]] name = "aes" version = "0.8.4" @@ -19,8 +38,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", - "cpufeatures", + "cipher 0.4.4", + "cpufeatures 0.2.17", ] [[package]] @@ -29,9 +48,9 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ - "aead", + "aead 0.5.2", "aes", - "cipher", + "cipher 0.4.4", "ctr", "ghash", "subtle", @@ -93,12 +112,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base64" version = "0.22.1" @@ -126,6 +139,25 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -139,8 +171,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", - "cipher", - "cpufeatures", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cipher 0.5.2", + "cpufeatures 0.3.1", ] [[package]] @@ -149,24 +192,47 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ - "aead", - "chacha20", - "cipher", - "poly1305", + "aead 0.5.2", + "chacha20 0.9.1", + "cipher 0.4.4", + "poly1305 0.8.0", "zeroize", ] +[[package]] +name = "chacha20poly1305" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" +dependencies = [ + "aead 0.6.1", + "chacha20 0.10.2", + "cipher 0.5.2", + "poly1305 0.9.1", +] + [[package]] name = "cipher" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", - "inout", + "crypto-common 0.1.7", + "inout 0.1.4", "zeroize", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.2", +] + [[package]] name = "clap" version = "4.6.6" @@ -207,6 +273,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -229,15 +301,12 @@ dependencies = [ ] [[package]] -name = "crypto-bigint" -version = "0.5.5" +name = "cpufeatures" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", + "libc", ] [[package]] @@ -251,13 +320,33 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + [[package]] name = "ctr" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", ] [[package]] @@ -267,7 +356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -303,8 +392,8 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] @@ -333,25 +422,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest", - "ff", - "generic-array", - "group", - "hkdf", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "errno" version = "0.3.14" @@ -368,22 +438,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "fiat-crypto" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + [[package]] name = "fs2" version = "0.4.3" @@ -402,7 +468,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -437,6 +502,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -453,9 +519,10 @@ dependencies = [ name = "git-remote-e2ee" version = "0.1.0" dependencies = [ + "aead-stream", "anyhow", "base64", - "chacha20poly1305", + "chacha20poly1305 0.11.0", "clap", "ed25519-dalek", "fs2", @@ -464,6 +531,7 @@ dependencies = [ "hpke", "rand 0.8.7", "rand 0.9.5", + "rand_chacha 0.9.0", "serde", "serde_json", "sha2", @@ -473,17 +541,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "heck" version = "0.5.0" @@ -520,14 +577,13 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f65d16b699dd1a1fa2d851c970b0c971b388eeeb40f744252b8de48860980c8f" dependencies = [ - "aead", + "aead 0.5.2", "aes-gcm", - "chacha20poly1305", + "chacha20poly1305 0.10.1", "digest", "generic-array", "hkdf", "hmac", - "p256", "rand_core 0.9.5", "sha2", "subtle", @@ -535,6 +591,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "inout" version = "0.1.4" @@ -544,6 +609,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -592,16 +666,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "elliptic-curve", - "primeorder", -] - [[package]] name = "pkcs8" version = "0.10.2" @@ -618,9 +682,19 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", +] + +[[package]] +name = "poly1305" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" +dependencies = [ + "cpufeatures 0.3.1", + "universal-hash 0.6.1", ] [[package]] @@ -630,9 +704,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", ] [[package]] @@ -644,15 +718,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -742,6 +807,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rustc_version" version = "0.4.1" @@ -764,19 +835,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "subtle", - "zeroize", -] - [[package]] name = "semver" version = "1.0.28" @@ -833,10 +891,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", + "sha2-asm", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signature" version = "2.2.0" @@ -941,10 +1015,20 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "utf8parse" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index d9bd6ba..e4a59e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,18 +21,20 @@ path = "src/bin/git-remote-e2ee.rs" [dependencies] anyhow = "1.0" base64 = "0.22" -chacha20poly1305 = { version = "0.10", features = ["stream"] } +aead-stream = "0.6" +chacha20poly1305 = "0.11" clap = { version = "4.5", features = ["derive"] } ed25519-dalek = { version = "2.1", features = ["rand_core"] } fs2 = "0.4" hex = "0.4" -hpke = { version = "0.13", features = ["x25519"] } +hpke = { version = "0.13", default-features = false, features = ["alloc", "x25519"] } hkdf = "0.12" rand = "0.8" rand_09 = { package = "rand", version = "0.9", features = ["os_rng", "std_rng"], default-features = false } +rand_chacha = "0.9" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sha2 = "0.10" +sha2 = { version = "0.10", features = ["asm"] } subtle = "2.6" thiserror = "2.0" tempfile = "3.14" diff --git a/README.md b/README.md index c3787ae..d7011b9 100644 --- a/README.md +++ b/README.md @@ -1,389 +1,392 @@ -# git-remote-e2ee - -[![CI](https://github.com/combinatrix-ai/git-remote-e2ee/actions/workflows/ci.yml/badge.svg)](https://github.com/combinatrix-ai/git-remote-e2ee/actions/workflows/ci.yml) -[![License: MIT OR Apache-2.0](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-blue.svg)](#license) - -End-to-end encrypted Git remotes on storage you do not have to trust with your -repository contents or Git metadata. - -`git-remote-e2ee` is a Git remote helper. You keep using ordinary `git clone`, -`git fetch`, `git pull`, and `git push`; the helper turns Git packs and refs into -authenticated ciphertext before they leave the client. The storage provider -sees opaque objects and an opaque latest-manifest pointer, not the inner branch -names, commit IDs, paths, authors, or messages. +

git-remote-e2ee

+ +

+ Git your host can't read.
+ End-to-end encrypted Git remotes. Keep using clone, pull, and push; + GitHub, a NAS, or any other storage only ever holds ciphertext. +

+ +

+ CI + License: MIT OR Apache-2.0 + Status: prototype +

+ +

+ Quick start + · How it compares + · Design + · Specification +

+ +

+ On your devices the repository is ordinary Git; the storage host only holds opaque encrypted objects +

> [!WARNING] -> This is an early prototype, not yet a safe backup system. The repository -> format is unstable and there are no compatibility guarantees between -> versions. Keep an independent copy of every repository and key. - -The implemented v4 protocol gives every published generation a fresh random -root key. Each active reader receives one small public-key envelope for that -generation; pack ciphertext is stored only once, independent of reader count, -and is authenticated in bounded-memory 1 MiB segments. -See the [design overview](docs/design.md) and [normative specification](docs/spec.md). - -## Why this exists - -Git hosting normally requires the server to understand the repository. That is -useful for web diffs, pull requests, search, and CI, but it also exposes the -entire object graph and most repository metadata to the host. - -`git-remote-e2ee` deliberately gives up server-side Git features. Git semantics -stay on trusted clients, while the server is reduced to two jobs: - -1. store immutable ciphertext objects; -2. atomically replace one opaque head value if it has not changed. - -This differs from [`git-crypt`](https://github.com/AGWA/git-crypt), which is -designed to encrypt selected files inside an otherwise normal repository. It is -closer to -[`git-remote-gcrypt`](https://github.com/spwhitton/git-remote-gcrypt), but uses -modern authenticated encryption, a signed policy and manifest chain, -per-device keys, incremental Git packs, and a backend-neutral compare-and-swap -storage contract. In -particular, the carrier-Git backend does not require uploading the entire inner -repository on every update. - -## Comparison with existing tools - -These projects solve two different problems. File-filter tools keep an ordinary -Git repository useful to its host while hiding selected blob contents. Encrypted -remote helpers hide the repository as a whole, which also removes server-side -diffs, pull requests, search, and CI over the plaintext. - -| | [`git-crypt`](https://github.com/AGWA/git-crypt) | [`transcrypt`](https://github.com/elasticdog/transcrypt) | [`git-remote-gcrypt`](https://github.com/spwhitton/git-remote-gcrypt) | `git-remote-e2ee` | -|---|---|---|---|---| -| Primary use | Encrypt selected files | Encrypt selected files | Encrypt a complete Git remote | Encrypt a complete Git remote | -| Integration | Git clean/smudge filters | Git clean/smudge filters | Git remote helper | Git remote helper | -| Hidden from host | Selected blob contents | Selected blob contents | Inner objects, refs, and encrypted manifest contents | Inner objects, refs, paths, authors, messages, and manifest contents | -| Host retains normal Git features | Yes, for visible repository data | Yes, for visible repository data | No | No | -| Update granularity | Per encrypted file; a changed encrypted file is stored again | Per encrypted file | Backend-dependent; Git and SFTP backends may retransmit full history | Incremental Git packs on filesystem and carrier-Git backends | -| Integrity model | Git repository integrity plus deterministic encrypted blobs | Git repository integrity plus encrypted blobs | Encrypted and signed manifest; ciphertext-addressed packs | AEAD packs/manifests, signed policy and manifest chains, and per-client history pinning | -| Key and collaborator model | Symmetric key or GPG users | Shared passphrase | GPG participants and symmetric mode | Per-repository device keys; any authorized device can decrypt independently; writers and administrators are separate roles | -| Maturity | Established | Established | Established | Experimental prototype | - -The closest comparison is `git-remote-gcrypt`. It already supports participant -management and several transports, making it the more mature choice today. -`git-remote-e2ee` is exploring a different storage protocol: immutable -incremental packs plus an explicit compare-and-swap head, modern AEAD, and a -signed history chain that returning clients pin locally. According to -`git-remote-gcrypt`'s -[`PERFORMANCE` documentation](https://manpages.debian.org/trixie/git-remote-gcrypt/git-remote-gcrypt.1.en.html#PERFORMANCE), -its arbitrary Git and SFTP transports upload the complete repository history on -each push; its rsync backend behaves differently. The comparison is therefore -backend-specific, not a claim that every `git-remote-gcrypt` update is a full -upload. - -If you need to protect a few secrets while retaining GitHub or GitLab features, -use a file-filter tool. If the storage provider must not learn the repository -structure or metadata, use a whole-remote encryption design—and, for now, treat -this project as research-grade software. - -## Current features - -- Normal Git remote-helper workflow for clone, fetch, pull, and push -- Per-device HPKE (X25519/HKDF-SHA-256/ChaCha20-Poly1305) generation-key envelopes -- A fresh random generation root for every successful HEAD publication -- Domain-separated HKDF subkeys and XChaCha20-Poly1305 payload encryption -- Streaming authenticated pack encryption and decryption with bounded Rust-side - memory rather than whole-pack buffers -- Ed25519-signed, append-only policy and manifest chains -- 1-of-N recipient access: each authorized device unlocks with only its own key -- Separate reader, writer, and administrator authorization -- Atomic device addition and revocation without rewriting historical packs -- An authenticated backward key chain: the current key unlocks earlier - generations, while an earlier key cannot unlock later generations -- Delta manifests and incremental Git packs rather than full repository snapshots -- Client-side fast-forward enforcement and explicit force push -- Atomic stale-writer rejection through compare-and-swap -- Per-client rollback and manifest-fork detection after first observation -- Complete signed history, ciphertext, and reconstructed Git object-graph - verification before refs or continuity pins move -- Filesystem storage backend -- Carrier-Git backend for GitHub, GitLab, a bare repository, or another ordinary - Git remote - -Push destinations are currently limited to branches under `refs/heads/*`. Tags -and branch deletion are rejected without publishing a new manifest. - -## Install - -The project currently builds two binaries: - -- `git-e2ee`: repository initialization, key generation, and diagnostic CLI -- `git-remote-e2ee`: helper invoked automatically by Git for `e2ee::` URLs - -```console -cargo install --path . -git-e2ee --help -``` - -Both binaries must be on `PATH` for native Git integration. - -## Quick start: filesystem backend - -Create a repository key and encrypted storage directory: - -```console -git-e2ee keygen --output /safe/place/repository.key.json -git-e2ee init \ - --storage /srv/encrypted/example \ - --key /safe/place/repository.key.json -``` - -Add it to an existing local Git repository: - -```console -git remote add private 'e2ee::/srv/encrypted/example' -git config remote.private.e2ee-key /safe/place/repository.key.json -git push -u private main -git fetch private -``` - -The key path is local Git configuration. The key file is never written to the -encrypted remote and must never be committed. - -## Add and revoke devices - -Each device has a different private key. Only the small public device file is -given to an administrator. If machine A created the repository, set up machine -B like this (substitute the repository root printed by A's `keygen`): - -```console -# Machine B -git-e2ee keygen \ - --repository-root \ - --output /safe/place/machine-b.key.json -git-e2ee device-export \ - --key /safe/place/machine-b.key.json \ - --output machine-b.public.json - -# Machine A, after receiving only machine-b.public.json -git-e2ee device-add \ - --storage /srv/encrypted/example \ - --key /safe/place/repository.key.json \ - --device machine-b.public.json -``` - -The default added device can read and write but cannot change policy. Pass -`--admin` to grant administration too. `git-e2ee device-list` prints opaque -device IDs; revoke one with: +> This is an early prototype. The repository format is unstable and may change +> without a migration path. Do not use it as your only copy of anything: keep an +> independent copy of every repository and every key. + +## Why end-to-end encryption? + +A private repository is private from other people, not from the host. Even +with encryption at rest, the host keeps the key next to the data, so whoever +breaks into the host, its storage, or its backups can read your code. +End-to-end encryption keeps the key on your devices: an intruder finds only +locked boxes, and anything they change is rejected. + +

+ Left: with a private repository your files sit on GitHub or GitLab in readable form, so a hacker who breaks in reads everything. Right: with git-remote-e2ee the host holds only locked boxes and the key stays on your laptop, so a hacker who breaks in finds nothing readable. +

+ +- **Good fits:** notes and journals, unpublished research, client or NDA work, + private configuration, backups on storage you do not control. +- **What you give up:** no web diffs, pull-request reviews, code search, or + hosted CI on the plaintext, and nobody can recover your data if you lose + every key. + +## Why git-remote-e2ee + +Encrypting Git is not new. File-level tools such as +[`git-crypt`](https://github.com/AGWA/git-crypt) encrypt selected files, and +[`git-remote-gcrypt`](https://github.com/spwhitton/git-remote-gcrypt) has +encrypted whole repositories with GnuPG for years. `git-remote-e2ee` hides the +whole repository like gcrypt, and fixes what makes that painful to live with: + +- **Fast: pushes upload only what changed.** A push uploads the new Git pack + plus a few KB of metadata on every backend: about 9 KB for a one-line + change in the [benchmark](#performance). With a Git or SFTP backend such as + GitHub, gcrypt re-uploads the entire encrypted history on every push: 921 MB + for the same change. +- **Safe: no silent force pushes.** Fast-forward checks run on the client, and + the storage moves `HEAD` only by compare-and-swap. When two people push at + once, one wins and the other gets an ordinary rejection. With gcrypt every + push is effectively a force push, so a push made without pulling first can + erase someone else's work. +- **No master key: nothing long-lived is shared.** git-crypt protects a + repository with one symmetric key that every collaborator ends up holding, + and it cannot revoke anyone. `git-remote-e2ee` creates a fresh random key for + every push and wraps it separately for each authorized device's own key. An + administrator can revoke one device without rewriting history, and it + receives nothing built on top of that revocation. + +It also does the rest of what an encrypted remote should: + +- **Hides who has access.** Device keys, roles, and membership changes are + encrypted. Storage sees only a padded device count, rounded up to a power of + two. +- **Read, write, and admin roles.** Give a CI runner or a reviewer a read-only + key. Only administrators can add or revoke devices. +- **Detects tampering, rollback, and forks.** A clone that has synced before + refuses an older or diverging state, and `git-e2ee recover` lets a writer + continue explicitly after someone corrupts `HEAD`. +- **Behaves like Git.** Branches, tags, remote branch deletion, and explicit + force pushes all work, and the remote grows only by what you add: 878 MiB + after five pushes, against 5.2 GiB for gcrypt. +- **Stays light.** A small push needs about 9 MiB of memory and takes about a + second. +- **Needs nothing else.** Two Rust binaries with all cryptography built in, no + GnuPG or OpenSSL. Storage can be any Git host (GitHub, GitLab, a bare + repository) or a local or mounted directory. + +Under the hood, Git runs on your machine as usual. The helper encrypts the +packs and refs Git hands it, uploads them as opaque immutable objects, and then +atomically moves one opaque `HEAD` pointer. + +### At a glance + +Godot repository (867 MiB of history), a local bare repository as the Git +host, Apple M1 Pro, median of three runs. Bold marks where `git-remote-e2ee` +beats `git-remote-gcrypt`. Full results in [Performance](#performance). + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial push | 28.4 s | 36.6 s | **22.9 s** | +| Fresh fetch | 27.7 s | 55.0 s | **40.0 s** | +| Tiny push | 0.08 s | 6.5 s | **0.97 s** | +| Tiny update (fetch) | 0.07 s | 22.6 s | **0.77 s** | +| Data sent per tiny push | 4.5 KB | 921 MB | **8.9 KB** | + +## Quick start + +Install from source (a Rust toolchain is required). This installs both +`git-e2ee`, the setup CLI, and `git-remote-e2ee`, the helper Git calls for +`e2ee::` URLs. Both must be on `PATH`. ```console -git-e2ee device-revoke \ - --storage /srv/encrypted/example \ - --key /safe/place/repository.key.json \ - --device-id +cargo install --git https://github.com/combinatrix-ai/git-remote-e2ee ``` -For a carrier-Git backend, use `--remote ` instead of `--storage` -with `device-add`, `device-list`, and `device-revoke`. - -Adding or revoking a device publishes a fresh generation key in one -compare-and-swap operation. Its header contains one small envelope for each -active reader, while all historical manifests and packs stay unchanged. A newly -added reader can use the current key's authenticated backward links to decrypt -the complete history. A revoked reader retains the snapshot it could already -decrypt but receives no key for the new generation or later ones. -The CLI keeps an administrative continuity pin next to the administrator key as -`.admin-state.json`. Preserve that file together with the key. - -This is deliberately **not multisig**. The current implementation accepts one -authorized administrator signature for a policy change. The wire format has an -administrator threshold and signature array so a future version can add M-of-N, -but this version fails closed on any threshold other than 1. - -## Carrier-Git backend - -An ordinary Git repository can act as the ciphertext carrier. It may be empty -or contain unrelated branches; `git-remote-e2ee` uses its own -`git-remote-e2ee` branch. +Use an empty GitHub repository as encrypted storage and push an existing +project to it: ```console -git-e2ee keygen --output /safe/place/carrier.key.json +git-e2ee keygen --output ~/.config/git-e2ee/notes.key.json git-e2ee carrier-init \ - --remote https://git.example/user/encrypted-carrier.git \ - --key /safe/place/carrier.key.json + --remote https://github.com/you/notes-encrypted.git \ + --key ~/.config/git-e2ee/notes.key.json -git remote add private \ - 'e2ee::git+https://git.example/user/encrypted-carrier.git' -git config remote.private.e2ee-key /safe/place/carrier.key.json +cd my-notes +git remote add private 'e2ee::git+https://github.com/you/notes-encrypted.git' +git config remote.private.e2ee-key ~/.config/git-e2ee/notes.key.json git push -u private main ``` -Clone by supplying the key path once: - -```console -git -c e2ee.key=/safe/place/carrier.key.json clone \ - 'e2ee::git+https://git.example/user/encrypted-carrier.git' -``` - -The helper persists the path as local `remote.origin.e2ee-key` configuration in -the new clone. Later `git fetch`, `git pull`, and `git push` need no wrapper. - -The carrier stores only the outer structure below: - -```text -refs/heads/git-remote-e2ee -└── e2ee/ - ├── HEAD - ├── objects/aa//00000000 - ├── manifests/bb//00000000 - └── policies/cc//00000000 -``` - -Ciphertext is divided into 32 MiB chunks so it can be carried as ordinary Git -blobs. The host can still observe outer commit times, chunk counts and sizes, -update frequency, and total growth. +From then on, `git pull` and `git push` work as usual. The key file stays on +your machine; never commit it, and keep a backup somewhere safe. + +To use a second machine, give it its own key and authorize its public half. +The [user guide](docs/guide.md#add-a-second-device) walks through it, as well as +the directory backend, cloning, and revoking devices. + +## How it compares + +There are two kinds of encrypted-Git tools. **File-level tools**, represented +here by [`git-crypt`](https://github.com/AGWA/git-crypt), encrypt the contents +of selected files inside an otherwise normal repository. **Encrypted remotes**, +[`git-remote-gcrypt`](https://github.com/spwhitton/git-remote-gcrypt) and +`git-remote-e2ee`, encrypt the whole repository, history included. + +### What the storage host can learn + +| | Private repo | `git-crypt` | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|:---:|:---:|:---:|:---:| +| Contents of files you chose to protect | visible | hidden | hidden | hidden | +| Contents of all other files | visible | visible | hidden | hidden | +| File names and directory layout | visible | visible | hidden | hidden | +| Commit messages, authors, and dates | visible | visible | hidden | hidden | +| Branch names and the commit graph | visible | visible | hidden | hidden | +| Which files changed, their sizes, identical files | visible | visible¹ | hidden | hidden | +| When you push, and roughly how much | visible | visible | visible | visible | +| Collaborators | account list | key fingerprints² | count only³ | count only⁴ | + +### What each tool can do + +| | `git-crypt` | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|:---:|:---:|:---:| +| Normal `clone`, `pull`, and `push` | ○ | ○ | ○ | +| GitHub or GitLab as storage | ○ | ○ | ○ | +| Hosted diffs, review, and search for unencrypted parts | ○ | × | × | +| Encrypt only selected files | ○ | × | × | +| Push uploads only new data | △⁵ | △⁶ | ○ | +| Concurrent pushes can't silently overwrite each other | ○ | ×⁷ | ○ | +| Tampering detected | △⁸ | ○ | ○ | +| No master key | ×⁹ | ○ | ○ | +| Revoke one collaborator | ×¹⁰ | ×¹¹ | ○¹² | +| Roles (read / write / admin) | × | ×¹³ | ○ | +| Detects a rolled-back or forked remote | × | × | △¹⁴ | +| No external dependencies | △¹⁵ | ×¹⁶ | ○ | +| Tags and remote branch deletion | ○ | ○ | ○ | +| Mature, stable format | ○ | ○ | × | +| Cryptography | AES-256-CTR, HMAC-SHA1 SIV | OpenPGP (GnuPG) | XChaCha20-Poly1305, HPKE (X25519), Ed25519 | + +○ supported · △ with caveats · × not supported + +
+Notes + +1. git-crypt's README states that it does not hide when a file changes, its + length, or whether two files are identical. +2. In git-crypt's GPG mode, the repository key is encrypted to each user and + committed under `.git-crypt/`, named by key fingerprint. +3. gcrypt hides recipient key IDs by default (`gpg -R`); the number of + encrypted-key packets is still observable. +4. The envelope count is padded to the next power of two, with a minimum of + four, so storage sees only an upper bound on the number of devices. Public + keys, roles, and membership changes are encrypted, though size patterns may + suggest a membership change. Do not reuse device keys between repositories. +5. A changed encrypted file becomes a new, unrelated ciphertext blob, so Git + cannot delta-compress it against the previous version. +6. gcrypt is incremental with its local and rsync backends, but its + [documentation](https://manpages.debian.org/trixie/git-remote-gcrypt/git-remote-gcrypt.1.en.html) + says a Git or SFTP backend uploads the entire history on every push, and it + may repack the remote without warning. +7. Every gcrypt push is effectively a force push. Its explicit-force option + prevents accidents but does not coordinate concurrent writers. +8. git-crypt authenticates encrypted file contents; file names, history, and + which files are encrypted are ordinary Git data. +9. A master key here means one long-lived key that every collaborator ends up + holding. git-crypt uses one symmetric repository key for the life of the + repository: GPG mode wraps that same key to each user's GPG key, and + without GPG everyone shares the same exported key file. gcrypt and + git-remote-e2ee encrypt each push under fresh keys delivered to each + recipient's own key. +10. git-crypt's README states that it does not support revoking access. +11. A participant can be dropped from `gcrypt.participants`, but gcrypt does + not document a revocation workflow. +12. Revocation publishes a fresh key without rewriting history. It cannot take + back data the device already had. +13. gcrypt's recipient list is the local `gcrypt.participants` setting of + whoever pushes, so any participant who can push decides who can read the + next state. +14. A clone that has synced before rejects rollback or a diverging history. A + brand-new clone cannot tell without an external anchor; see + [Security model](#security-model). +15. git-crypt is a C++ program linked against OpenSSL; GPG mode also needs + GnuPG. +16. gcrypt is a shell script that requires GnuPG. + +
+ +### When another tool is a better fit + +- **git-remote-gcrypt**: you want an established tool with a stable format, + already use GPG, and work alone or can coordinate pushes. +- **A file-level tool such as git-crypt**: only a few files are secret, and you + want GitHub or GitLab to keep working normally for the rest. +- **An ordinary private repository**: you trust the host and want pull + requests, search, previews, and CI. ## Security model -The storage provider is treated as malicious for confidentiality and integrity. -Authenticated encryption, ciphertext-addressed objects, signatures, and the -manifest hash chain detect modification. A returning client pins the newest -manifest it has observed and rejects a lower generation or a history that no -longer descends from that pin. - -The current security claim is: - -- confidentiality of inner repository contents and Git metadata; -- authenticity and integrity of fetched repository state; -- continuity from the state previously observed by the same local clone. -- future-generation exclusion after an atomic device revocation. - -It does **not** independently prevent: - -- rollback presented to a fresh client; -- equivocation between clients shown different valid histories; -- freezing a client at its last valid state; -- deletion or denial of service by storage; -- destructive changes made by an authorized writer; -- disclosure of data a revoked device could access before its revocation; -- disclosure of the complete historical snapshot at or before any leaked - generation key; -- a revoked writer and colluding storage presenting a pre-revocation fork to a - fresh or stale client; -- policy rollback, freezing, or equivocation presented to a fresh client. - -Global rollback and equivocation resistance require gossip or an external -transparency anchor. A returning clone pins both manifest and policy generation. -The administrative CLI also pins its last published state and does not -automatically retry a lost CAS race; inspect the winner and rerun the operation. - -Generation headers and policy objects must remain plaintext-structured so a -device can find its envelope without already knowing the generation key. -Consequently the storage host can see device count, per-repository public keys, -roles, policy changes, and generation numbers. Device keys should never be -reused between repositories. Inner refs, object IDs, paths, authors, messages, -generation keys, derived subkeys, and contents remain encrypted. - -A generation key is intentionally a transferable snapshot capability: leaking -`K_t` exposes generations `0..=t` through the backward links. It does not expose -generation `t+1` or later. Continued future disclosure therefore requires a -reader's private device key, repeated release of each later generation key or -plaintext, or continued access to an authorized device. This is key regression, -not forward secrecy for already published history. - -If two authorized writers race, each may locally create a valid generation key -and ciphertext, but compare-and-swap permits only one HEAD update. The loser -must discard its unpublished generation and retry from the winning HEAD. Anyone -who received the loser's key can decrypt that losing unpublished snapshot; CAS -prevents it from becoming repository history but cannot retract already shared -plaintext or keys. - -## Storage protocol - -The backend-neutral interface has four logical operations. Immutable object -writes are staged so the final ciphertext ID can be learned while streaming: - -```text -begin_object(kind) -> writable stage; stage.finish(id) -open_object(kind, id) -> reader -read_head() -compare_and_swap_head(expected, next) -``` - -The filesystem backend implements head CAS with an advisory lock and a rename -in the storage root. Object publication hard-links a staged file from -`.staging` into `objects//` on the same filesystem and does not replace -an existing id. File contents are flushed before the name is published, new -directories are flushed through the preexisting ancestor, and the parent -directory is flushed again afterwards. A flush error fails the call. Stages -stay inside the backend's own filesystem or checkout, so publication does not -cross filesystems. Crash and power-loss limits are in -[docs/durability.md](docs/durability.md). The carrier-Git backend -implements head CAS as a normal fast-forward push to the outer branch. A future -S3 backend can use multipart upload plus conditional writes, but each provider -must be capability-tested; “S3 compatible” does not by itself promise correct -compare-and-swap behavior. - -A killed filesystem writer can leave an unreachable file under `.staging/`. -No published object or `HEAD` points to it, and stale `.stage-*` entries may be -deleted when no writer is running. Automatic age-based cleanup belongs to -future garbage collection. - -See [docs/design.md](docs/design.md) for the protocol and threat-model details. - -## Tests - -```console -cargo test --all-targets -cargo clippy --all-targets -- -D warnings -cargo fmt --all -- --check -``` - -The ignored power-cut harness is not part of that default run. Protocol, -Windows commands, and durability limits are in [docs/durability.md](docs/durability.md). - -The test suite includes: - -- incremental push and reconstruction into a fresh repository; -- native clone, fetch, pull, push, dry-run, refspec, and force-push behavior; -- rollback, same-generation fork, and ciphertext-tampering rejection; -- streaming-AEAD chunk boundaries, wrong keys/AAD, reordering, duplication, - truncation, trailing bytes, forged sizes, and legacy-format rejection; -- independent device add/read/write, non-admin rejection, genesis-substitution - rejection, administrative rollback pinning, device revocation without pack - rewrites, and multi-generation offline catch-up; -- malformed predecessor-link, generation-key commitment, recipient-set, and - signed-but-incomplete Git object-graph rejection; -- two independent carrier writers racing real `git push` processes, with - exactly one winner; -- plaintext-absence checks across carrier history; -- applicable black-box scenarios independently reimplemented from Git - upstream's - [`t/t5801-remote-helpers.sh`](https://github.com/git/git/blob/master/t/t5801-remote-helpers.sh). - -A local bare Git repository is sufficient for deterministic carrier concurrency -tests: local-path pushes still execute Git's real `receive-pack`, ref locking, -and fast-forward checks. Hosted smoke tests remain useful for authentication, -request limits, and provider-specific policy. - -## Roadmap - -- M-of-N administrative authorization (future format version; threshold 1 only today) -- Recovery and device-key replacement workflows -- Automatic stale-push fetch/retry workflow -- Persistent partial-clone cache for large carrier repositories -- S3 conditional-write backend and provider compatibility suite -- Safe compaction and garbage collection -- Signed checkpoint transitions for compaction without ambiguous key-chain semantics -- Optional gossip or transparency-log anchoring -- Shallow and partial clone support - -## Performance benchmarks - -An opt-in local harness measures initial encryption, fresh reconstruction, -verification, and incremental updates without contacting the source -repository's configured remote. See [docs/benchmarks.md](docs/benchmarks.md). Benchmark -outputs, repository keys, and reconstructed data must not be committed. +The storage provider is treated as malicious. `git-remote-e2ee` aims to +guarantee: + +- confidentiality of inner contents and Git metadata: files, paths, refs, + object IDs, authors, and messages; +- authenticity and integrity of every fetched state: only devices with write + authority under the authenticated policy history can author a state that + conforming clients accept; +- continuity for a clone that has synced before: it rejects any state older + than, or diverging from, its continuity floor, the newest state it accepted + after a complete connectivity check or its own successful push; +- that a revoked device gets no keys for generations that conforming writers + build on top of its revocation. + +Write authority is about what clients accept, not about who can put bytes on +the storage. Anyone who can write to the storage, including a read-only member +or someone with only push access to the carrier repository, can still replay +old valid states, hide updates, or corrupt `HEAD`. Clients reject what they +cannot verify, so this can deny service but cannot make unauthorized history +accepted. Restrict storage write access as a separate layer, for example with +read-only repository permissions and branch protection on GitHub. + +It does **not** prevent: + +- the host or anyone with storage write access deleting data, corrupting + `HEAD`, or refusing service; +- a brand-new clone, or a clone that has not synced since, being shown an old + or forked history, or different clients being shown different histories + (that needs an external anchor, which is on the roadmap); +- a revoked writer or administrator creating a valid-looking fork from before + its revocation, which clients that never saw the revocation cannot tell apart; +- an authorized writer making destructive changes; +- a revoked device reading what it could read before revocation; +- traffic analysis: update times, object sizes, total growth, and the padded + reader count are visible; +- future quantum attacks: key exchange uses X25519. + +Concurrent pushes cannot silently overwrite each other as long as the storage +honors compare-and-swap, as a Git host's fast-forward check or the directory +backend's lock does. + +Keys work as a backward chain. The current key can decrypt all earlier +history, which lets a new device read the whole repository, but an old key +cannot decrypt anything newer. This is key regression, not forward secrecy for +history already published. See [docs/design.md](docs/design.md) and [docs/spec.md](docs/spec.md) +for the full threat model, and [SECURITY.md](SECURITY.md) to report an issue. + +## Performance + +Committing is ordinary Git: encryption happens only when you push or fetch. +Small pushes and fetches upload or download only the change, even when GitHub +or another Git host is the storage. + +Measured on the Godot repository (867 MiB of history), with a local bare +repository standing in for the Git host, on an Apple M1 Pro with 32 GiB RAM. +Median of three runs; tiny rows are a one-line change, repeated five times. +Bold marks where `git-remote-e2ee` beats `git-remote-gcrypt`. + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial encryption (to a local directory) | 28.6 s | 11.0 s | **8.1 s** | +| Initial push | 28.4 s | 36.6 s | **22.9 s** | +| Fresh fetch | 27.7 s | 55.0 s | **40.0 s** | +| Tiny commit | 0.09 s | 0.09 s | 0.09 s | +| Tiny push | 0.08 s | 6.5 s | **0.97 s** | +| Tiny update (fetch) | 0.07 s | 22.6 s | **0.77 s** | +| Data sent per tiny push | 4.5 KB | 921 MB | **8.9 KB** | +| Peak memory, initial push | 1.27 GiB | 0.98 GiB | 1.27 GiB | +| Peak memory, tiny push | 37 MiB | 887 MiB | **9 MiB** | +| Remote size after five tiny pushes | 891 MiB | 5,272 MiB | **878 MiB** | +| Client disk after fresh fetch (`.git` plus cache) | 891 MiB | 1,780 MiB | 1,779 MiB | + +With a Git backend, gcrypt sends the whole encrypted history again on every +push, so the remote grows by about the repository size each time. +`git-remote-e2ee` sends the new pack plus a few KB of metadata. On the client, +`git-remote-e2ee` keeps a local cache of the encrypted carrier next to your +repository, which roughly doubles disk use, the same as gcrypt's local copy. +Peak memory for full transfers is dominated by Git's own pack generation. Plain Git +stays faster on small operations because a Git server understands the +repository; an encrypted remote has to verify and decrypt on the client. These +are local measurements without network latency. + +To reproduce these numbers, run `scripts/reproduce-benchmark.sh` from a clone +of this repository on macOS or Linux. It fetches the pinned inputs (about +1 GB), runs three rounds, and writes this table plus raw data and environment +details to `bench-results/`. Expect a few hours and about 12 GB of free disk. +[docs/benchmarks.md](docs/benchmarks.md) has the method, a Linux aarch64 +cross-check, per-push series, phase breakdown, and caveats. + +## FAQ + +**What does the GitHub repository look like?** +One branch, `git-remote-e2ee`, holding ciphertext under `e2ee/`. It can live +next to unrelated branches. Do not edit it by hand. + +**What if I lose my key?** +If no remaining device has access, the data is gone. There is no recovery +service by design. Keep an offline backup of at least one administrator key and +its `.admin-state.json` file. + +**Does revoking a device hide old history from it?** +No. It keeps whatever it could already decrypt. It gets no keys for anything +published on top of the revocation. A writer who never saw the revocation, or +was shown a stale state, could still publish to the old reader set; see +[Security model](#security-model). + +**Can I review pull requests?** +Not on the host. Review happens on a machine that has a key: fetch the branch +and diff locally, or give a CI runner its own read-only device key. + +**What if someone corrupts `HEAD`?** +Clients refuse it and stop. A writer can then run `git-e2ee recover`, which +shows what it found and continues from the last verified state, or from a newer +legitimate state it discovers in the carrier history. See the +[user guide](docs/guide.md#recover-from-a-broken-head). + +**Can the host roll my repository back?** +It can serve old data. A clone that has synced before refuses anything older +than, or diverging from, what it already accepted. A fresh clone, or a clone +that has not synced since a newer state was published, cannot tell yet. + +## Status and roadmap + +Working today: clone, fetch, pull, and push, including tags and remote branch +deletion; the directory and Git-host backends; incremental pushes and fetches +(the Git-host backend keeps a local cache of the carrier); per-device keys with +read, write, and admin roles; adding and revoking devices; rollback and race +detection; and explicit recovery from a corrupted `HEAD`. + +Planned: M-of-N administrator approval, key recovery and replacement, automatic +retry after losing a push race, an S3 conditional-write backend, garbage +collection and compaction, shallow and partial clone, optional +transparency-log anchoring, and an optional file-level mode that encrypts only +selected files while keeping per-device keys and revocation. + +## Documentation + +- [User guide](docs/guide.md): backends, cloning, device management, and the + storage layout +- [docs/design.md](docs/design.md): design overview and threat model +- [docs/spec.md](docs/spec.md): normative protocol specification +- [docs/benchmarks.md](docs/benchmarks.md): benchmark method and results +- [docs/durability.md](docs/durability.md): crash and power-loss guarantees of the filesystem backend +- [CONTRIBUTING.md](CONTRIBUTING.md): building, testing, and what the test + suite covers ## License -Licensed under either of - -- [Apache License, Version 2.0](LICENSE-APACHE), or -- [MIT License](LICENSE-MIT) - -at your option. +Licensed under either of the [Apache License, Version 2.0](LICENSE-APACHE) or +the [MIT License](LICENSE-MIT), at your option. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c8ea0bd --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security + +Please report vulnerabilities privately to +**git-remote-e2ee@combinatrix.ai** rather than opening a public issue. + +`git-remote-e2ee` is an early prototype and has not been independently +audited. Its intended guarantees and known limits are listed in the +[README](README.md#security-model) and specified in [docs/design.md](docs/design.md) and +[docs/spec.md](docs/spec.md). Reports are especially welcome about: + +- plaintext, refs, paths, or other inner metadata reaching storage; +- accepting tampered, rolled-back, or forked state that a returning clone + should reject; +- a revoked device obtaining keys for later generations; +- authorization bypass between reader, writer, and administrator roles; +- key material leaking into logs, Git configuration, or the remote. + +Limits already documented, such as rollback shown to a fresh clone or metadata +visible to storage, are not vulnerabilities by themselves. Proposals to close +them are still welcome as issues. diff --git a/docs/art/LICENSE-griddy-icons b/docs/art/LICENSE-griddy-icons new file mode 100644 index 0000000..79255e2 --- /dev/null +++ b/docs/art/LICENSE-griddy-icons @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Griddy Icons + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/art/LICENSE-tabler-icons b/docs/art/LICENSE-tabler-icons new file mode 100644 index 0000000..3e82379 --- /dev/null +++ b/docs/art/LICENSE-tabler-icons @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020-2026 Paweł Kuna + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/art/overview.svg b/docs/art/overview.svg new file mode 100644 index 0000000..1a4e913 --- /dev/null +++ b/docs/art/overview.svg @@ -0,0 +1,47 @@ + + What your devices see and what the storage host sees + On your devices the repository is ordinary Git with branches, commits, and files. It is encrypted before it leaves the device, so the storage host only holds opaque ciphertext objects and one opaque head pointer. + + + + + + On your devices + ordinary Git, fully readable + $ git log --oneline + a1f3c9e Fix login redirect + 7c02d41 Add customer notes + 3e9b7aa Initial import + $ ls + src/auth.rs notes/acme.md + branches: main, feature/sso + key: only on this device + + + + + encrypt + before + upload + + + + On the storage host + GitHub, a NAS, a bare repo… + e2ee/HEAD + 9f4c…e21a + e2ee/objects/ + 3f/9a1c07… (1.2 MiB) + 7c/e04b5d… (3.4 KiB) + b8/51f2aa… (80 KiB) + no names, paths, messages, + authors, or branches + diff --git a/docs/art/why-e2ee.svg b/docs/art/why-e2ee.svg new file mode 100644 index 0000000..2d60bff --- /dev/null +++ b/docs/art/why-e2ee.svg @@ -0,0 +1,48 @@ + + A hacker breaks into the Git host + Left, a private repository: your files sit on GitHub or GitLab in readable form, so a hacker who breaks in reads everything. Right, git-remote-e2ee: the host stores only locked boxes and the key stays on your laptop, so a hacker who breaks in finds nothing readable. Icons from Tabler Icons (MIT) and Griddy Icons (MIT). + + + + + + + + + + Private repository + + + + + + + GitHub, GitLab, … + + + + + + + + A break-in exposes everything. + Your files sit on the server in readable form. + + git-remote-e2ee + + + + + + + GitHub, GitLab, … + + + + + + + ? + A break-in finds only locked boxes. + The key never leaves your devices. + diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 8eb2ca8..d2b4dfd 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,4 +1,365 @@ -# Local performance benchmarks +# Benchmarks + +## Reproduce + +From a fresh clone on macOS or Linux, run: + +```sh +./scripts/reproduce-benchmark.sh +``` + +The first run downloads about 1 GB for Godot's full default-branch history and +the pinned gcrypt source. Keep free temporary disk space of about 12 times the +Godot `.git` size. The default three-round run takes hours. Results go to +`./bench-results/-/` and include raw and median TSV files, +`environment.json`, and a README-format `summary.md`. + +On Linux, the work directory defaults to `/var/tmp` because `/tmp` may be +tmpfs. The runner detects the selected filesystem and refuses tmpfs or ramfs. +Set `TMPDIR` to a disk-backed temporary directory, or set +`BENCH_WORK_PARENT` to a disk-backed path under `/var/tmp` or `TMPDIR`. + +The fresh-fetch client-disk row totals the client .git directory and any +auxiliary cache or state. + +## Latest reproduce run (2026-10-08) + +The README table comes from this run of `scripts/reproduce-benchmark.sh` with +its defaults (three rounds, five tiny commits), at e2ee `cedc9fe` with no +uncommitted changes, on an Apple M1 Pro (10 cores) with 32 GiB RAM, +macOS 26.5.2, Git 2.55.0, and Homebrew Rust 1.98.0. The machine was otherwise +lightly loaded. The runner's `summary.md`: + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial encryption (to a local directory) | 28.6 s | 11.0 s | 8.1 s | +| Initial push | 28.4 s | 36.6 s | 22.9 s | +| Fresh fetch | 27.7 s | 55.0 s | 40.0 s | +| Tiny commit | 0.09 s | 0.09 s | 0.09 s | +| Tiny push | 0.08 s | 6.5 s | 0.97 s | +| Tiny update (fetch) | 0.07 s | 22.6 s | 0.77 s | +| Data sent per tiny push | 4.5 KB | 921 MB | 8.9 KB | +| Peak memory, initial push | 1.27 GiB | 0.98 GiB | 1.27 GiB | +| Peak memory, tiny push | 37 MiB | 887 MiB | 9 MiB | +| Remote size after 5 tiny pushes | 891 MiB | 5,272 MiB | 878 MiB | +| Total client disk after fresh fetch (.git + auxiliary cache/state) | 891 MiB | 1,780 MiB | 1,779 MiB | + +### Second macOS machine (2026-10-08) + +The same runner at e2ee `971894a` on an Apple M1 Max with 64 GiB RAM, +macOS 27.0, and Git 2.56.0. That machine was shared with other work during the +run (load average about 7-16 on 10 cores). The ordering is identical: + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial encryption (to a local directory) | 30.3 s | 12.0 s | 8.9 s | +| Initial push | 30.3 s | 37.6 s | 23.9 s | +| Fresh fetch | 28.0 s | 53.5 s | 38.0 s | +| Tiny commit | 0.16 s | 0.16 s | 0.16 s | +| Tiny push | 0.09 s | 6.8 s | 1.2 s | +| Tiny update (fetch) | 0.07 s | 24.3 s | 0.84 s | +| Data sent per tiny push | 4.8 KB | 921 MB | 9.5 KB | +| Peak memory, initial push | 1.27 GiB | 0.96 GiB | 1.27 GiB | +| Peak memory, tiny push | 37 MiB | 887 MiB | 9 MiB | +| Remote size after 5 tiny pushes | 891 MiB | 5,272 MiB | 878 MiB | +| Total client disk after fresh fetch (.git + auxiliary cache/state) | 891 MiB | 1,780 MiB | 1,779 MiB | + +The sections below record earlier runs and the investigation that led here. + +## Primary Git-backend benchmark: Godot (2026-10-03) + +This compares plain Git over `file://` with `--no-local`, `git-remote-gcrypt` +using its Git backend (`gcrypt::file://...`), and E2EE using its carrier-Git +backend (`e2ee::git+file://...`). Initial encryption is a separate local +backend baseline: plain Git pushes to a local bare repository, while gcrypt +and E2EE use their local-directory backends. + +The source was `godotengine/godot` at `00932449c9f372b30301d8b5fdc1be70ec12b5c0` +(default branch only, no tags): 85,678 commits, 14,162 files at HEAD, and +910,290,704 reachable object bytes. The gcrypt helper was at +`a5ff704d071f14b95b6b1fa0caa8cdbf0c6cdadb`; the E2EE release binary was built +from `7036a37dda4eea16195e19429e5a5807ebd37858`. Each of three rounds used +fresh bare remotes, clients, cache directories, and a temporary GPG home. Git +auto-GC, maintenance, and receive auto-GC were disabled. The bare receivers +used `receive.unpackLimit=0`; the carrier receiver also used +`attr.tree=refs/heads/git-remote-e2ee`, so it could read the committed +`e2ee/** -delta` attribute. The E2EE push and returning-client caches were +isolated per round under the temporary benchmark directory. Each tiny commit +replaced one line in Godot's tracked README and was pushed and fetched before +the next. See `scripts/benchmark-git-backend.sh` for the complete harness. + +Machine: Apple M1 Max (10 CPU cores), 64 GiB RAM (`sysctl hw.memsize` = +68,719,476,736 bytes), macOS 27.0 build 26A428. Versions: Git 2.55.0, GnuPG +2.5.24, Rust 1.97.1. + +The harness wraps every measured command in macOS `/usr/bin/time -l` (or GNU +`/usr/bin/time -v` on Linux). Peak RSS is the platform-reported maximum for +that command and its waited-for children, where supported; it is not the sum +of concurrent process RSS. Logical directory size sums regular-file lengths. +Allocated size uses `du -sk` in 1 KiB blocks. `Remote / stored L/A` is logical +and allocated MiB for the local storage directory or bare remote after the +phase. `Client .git L/A` is the whole client Git directory after the phase. +`Auxiliary state / cache L/A` is gcrypt's `.git/remote-gcrypt` state or the +whole isolated E2EE cache; `—` means not applicable. The byte column is the +logical object-store increase for pushes and tiny updates, the stored amount +for initial encryption, and the remote object-store reference size for fresh +fetch. Fetch rows also include client object-store growth in the final column. +These are filesystem/object-database measurements, not packet-level wire +counts. Gcrypt's client `.git` contains both encrypted carrier objects and +imported inner Git objects. The sizes are workload-specific and the local +`file://` timings do not predict GitHub server throughput. +The harness processes used only run-specific caches under the temporary +benchmark directory. A separate short-lived Git helper against the default +user cache appeared in process listings during the run; it was left untouched, +and any timing effect from that concurrent activity is unquantified. + +Medians across three fresh rounds; repository sizes are MiB (`L/A` = logical / +allocated). The byte column remains exact bytes. + +| Phase | Transport | Time | Peak RSS | Object/stored bytes | Remote / stored L/A | Client `.git` L/A | Auxiliary state / cache L/A | Client object growth B | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| Initial encryption | Plain Git | 31.97 s | 1302.70 MiB | 934,460,946 | 891.20 / 903.80 | 892.77 / 904.83 | — | — | +| Initial encryption | gcrypt | 13.38 s | 983.23 MiB | 921,021,599 | 878.35 / 878.36 | 892.77 / 904.84 | — | — | +| Initial encryption | E2EE | 10.08 s | 1302.02 MiB | 920,636,512 | 877.99 / 880.02 | 892.77 / 904.85 | — | — | +| Initial push | Plain Git | 32.91 s | 1301.84 MiB | 934,460,946 | 891.20 / 903.42 | 892.77 / 904.86 | — | — | +| Initial push | gcrypt | 41.11 s | 983.28 MiB | 921,304,151 | 878.65 / 881.59 | 1771.39 / 1785.52 | — | — | +| Initial push | E2EE | 25.76 s | 1304.09 MiB | 920,777,562 | 878.15 / 880.31 | 1771.39 / 1785.53 | 878.18 / 892.33 | — | +| Fresh fetch | Plain Git | 30.83 s | 1306.02 MiB | 934,460,946 | 891.20 / 903.42 | 891.20 / 903.25 | — | 934,460,946 | +| Fresh fetch | gcrypt | 56.70 s | 886.61 MiB | 921,304,151 | 878.65 / 881.59 | 1779.70 / 1784.19 | 0.00 / 0.00 | 1,866,126,563 | +| Fresh fetch | E2EE | 41.09 s | 886.42 MiB | 920,764,311 | 878.15 / 880.31 | 901.08 / 903.70 | 878.14 / 878.37 | 944,824,608 | +| Tiny commit | Plain Git | 0.19 s | 13.53 MiB | 0 | 891.21 / 903.44 | 1771.41 / 1785.24 | — | — | +| Tiny commit | gcrypt | 0.19 s | 13.53 MiB | 0 | 2635.91 / 2656.59 | 1771.41 / 1785.24 | — | — | +| Tiny commit | E2EE | 0.19 s | 13.53 MiB | 0 | 878.17 / 880.34 | 1771.41 / 1785.24 | 878.20 / 892.42 | — | +| Tiny push (median) | Plain Git | 0.10 s | 37.47 MiB | 4,762 | 891.21 / 903.45 | 1771.41 / 1785.24 | — | — | +| Tiny push (median) | gcrypt | 7.29 s | 887.44 MiB | 921,315,032 | 3514.54 / 3548.37 | 1771.42 / 1785.27 | — | — | +| Tiny push (median) | E2EE | 1.47 s | 37.91 MiB | 9,462 | 878.18 / 880.36 | 1771.42 / 1785.27 | 878.21 / 892.47 | — | +| Tiny update (median) | Plain Git | 0.08 s | 37.33 MiB | 4,762 | 891.21 / 903.45 | 891.21 / 903.30 | — | 3,359 | +| Tiny update (median) | gcrypt | 25.41 s | 886.67 MiB | 921,315,032 | 3514.54 / 3548.37 | 1779.73 / 1784.28 | 0.00 / 0.00 | 9,472 | +| Tiny update (median) | E2EE | 1.04 s | 39.03 MiB | 9,462 | 878.18 / 880.36 | 901.10 / 903.74 | 878.16 / 878.51 | 4,499 | + +The size pairs are rounded to 0.01 MiB. Gcrypt's auxiliary state was 78 B +logical / 4,096 B allocated after fresh fetch and 312 B / 4,096 B after tiny +update; both display as `0.00 / 0.00 MiB` above. Compared with the prior +published run, E2EE's warm tiny-push and tiny-update medians moved from 1.23 s +/ 0.89 s to 1.47 s / 1.04 s. This is an observed difference across separate +three-round runs; the independent helper activity described above overlapped +the latest run, so its cause is unmeasured. + +The per-push medians below include wall time, peak RSS, and logical bytes +added to the bare remote (for fetches, remote bytes associated with that +update followed by client object-store growth). RSS is MiB. + +| Tiny push | Plain Git: time / RSS / remote bytes | gcrypt Git: time / RSS / remote bytes | E2EE carrier Git: time / RSS / remote bytes | +|---|---:|---:|---:| +| 1 (first) | 0.20 s / 37.3 MiB / 4,776 | 8.06 s / 887.1 MiB / 921,307,778 | 5.20 s / 522.3 MiB / 9,345 | +| 2 | 0.10 s / 37.9 MiB / 4,760 | 7.38 s / 887.1 MiB / 921,311,401 | 1.32 s / 38.0 MiB / 9,405 | +| 3 | 0.10 s / 37.9 MiB / 4,762 | 7.21 s / 887.1 MiB / 921,315,032 | 1.35 s / 37.9 MiB / 9,462 | +| 4 | 0.10 s / 37.9 MiB / 4,763 | 7.47 s / 887.2 MiB / 921,318,660 | 1.47 s / 37.9 MiB / 9,522 | +| 5 | 0.10 s / 38.0 MiB / 4,759 | 7.27 s / 887.2 MiB / 921,322,283 | 1.53 s / 37.9 MiB / 9,577 | + +| Tiny update fetch | Plain Git: time / RSS / remote bytes / client growth | gcrypt Git: time / RSS / remote bytes / client growth | E2EE carrier Git: time / RSS / remote bytes / client growth | +|---|---:|---:|---:| +| 1 (first) | 0.18 s / 37.4 MiB / 4,776 / 3,358 B | 25.95 s / 887.0 MiB / 921,307,778 / 9,010 B | 0.89 s / 39.0 MiB / 9,345 / 4,498 B | +| 2 | 0.08 s / 37.3 MiB / 4,760 / 3,359 B | 25.20 s / 887.0 MiB / 921,311,401 / 9,241 B | 0.92 s / 39.0 MiB / 9,405 / 4,500 B | +| 3 | 0.08 s / 37.3 MiB / 4,762 / 3,359 B | 25.33 s / 887.1 MiB / 921,315,032 / 9,472 B | 1.04 s / 39.1 MiB / 9,462 / 4,499 B | +| 4 | 0.09 s / 37.3 MiB / 4,763 / 3,359 B | 25.50 s / 887.1 MiB / 921,318,660 / 9,706 B | 1.04 s / 39.0 MiB / 9,522 / 4,501 B | +| 5 | 0.08 s / 37.3 MiB / 4,759 / 3,356 B | 25.41 s / 887.1 MiB / 921,322,283 / 9,927 B | 1.15 s / 39.1 MiB / 9,577 / 4,498 B | + +In the 2026-10-03 run, gcrypt added about 921 MiB to the remote on each tiny +push. E2EE's first tiny push took 5.20 s and used 522.3 MiB peak RSS. Later +pushes took 1.32 to 1.53 s, used about 38 MiB, and added 9.3 to 9.6 KiB. +E2EE's median tiny push and fetch were below 1.5 s and 1.1 s, respectively. + +#### First-push follow-up (2026-10-08) + +Commit `f4526d8` advances the verified refs after a local publication only +when the client's previous refs already match its verified set and the local +repository is neither shallow nor promisor-enabled. This removes the full +history connectivity walk from the first tiny push after an initial push. + +Both samples used the pinned Godot history, one round, and three tiny commits. +The before sample ran E2EE only. The after sample ran all three backends. + +| Tiny push | Before fix time | Before fix peak RSS | After fix time | After fix peak RSS | +|---|---:|---:|---:|---:| +| 1 | 4.33 s | 528.0 MiB | 1.10 s | 8.97 MiB | +| 2 | 1.08 s | 37.7 MiB | 1.17 s | 8.84 MiB | +| 3 | 1.14 s | 37.7 MiB | 1.32 s | 8.81 MiB | + +A separate pre-fix trace spent 3,335 ms in `git_ref_connectivity_walk` on +push 1. A post-fix trace checked zero objects in that walk and spent 71 ms +there. The Mac's one-minute load average was 11.87 on 10 cores before the +after run. These are single-round follow-up samples, not replacements for the +three-round table above. + +#### Initial-encryption profiling and changes + +The before and after profiles are traced single runs; the final result above +is the median of three untraced rounds. Phase timers overlap and must not be +summed. Before enabling accelerated crypto, the traced directory encryption +took 17.47 s: `git_pack_objects_stream_lifetime` 16.928 s, pack-pipe reads +8.276 s, ciphertext stage writes 3.254 s, SHA-256 2.919 s, ChaCha20-Poly1305 +2.451 s, and durable stage finish 0.033 s. With `sha2` hardware assembly and +the AArch64 ChaCha20 NEON backend, a traced run took 9.80 s: stream lifetime +9.399 s, pipe reads 6.411 s, stage writes 0.877 s, SHA-256 0.453 s, ChaCha +1.657 s, and stage finish 0.033 s. The final untraced median was 10.08 s. +Compared with the prior 14.20 s median, this is 4.12 s (29%) faster; in the +same final run, it beat gcrypt's 13.38 s by 3.30 s (25%). +E2EE used 1302.02 MiB peak RSS for this phase, 318.79 MiB more than gcrypt's +983.23 MiB; Git's inner `pack-objects` process dominates that peak. + +The source repository had no bitmap and no pack-related compression overrides. +The carrier's compression-free settings do not apply to the user's inner Git +repository. A cold standalone `pack-objects --stdout --revs` took 8.96 s. +For gcrypt's shape, `rev-list --objects` took 3.16 s and the following +`pack-objects --stdout` took 4.17 s; the intermediate object list was 46.5 MB +for 756,518 objects. On warm probes, the direct pipeline took 6.33 s versus +6.72 s for `pack-objects --stdout --revs`. Individual options also showed no +stable gain: `--delta-base-offset` took 6.40 s, `--threads=0` 6.37 s, and +`--path-walk` 6.56 s. There was no bitmap index for `--use-bitmap-index` to +use. Existing packed-object reuse remains enabled. A bounded writer-thread +experiment was reverted: its queue wait was only about 40 ms and the traced +end-to-end samples did not improve. +The retained change accelerates the existing SHA-256 and ChaCha20 operations; +the wire format, AAD, segment counters, durability steps, and Git pack +generation command are unchanged. + +#### End-user `cargo install` verification (2026-10-03) + +I built the pushed `64ae6e4` revision using `cargo install --git +https://github.com/combinatrix-ai/git-remote-e2ee --branch readme-rewrite` +from outside the checkout, with an isolated temporary `CARGO_HOME` and no +`RUSTFLAGS`. The exact invocation also used `--root --force +git-remote-e2ee` to isolate the installation. Cargo resolved +`chacha20poly1305 0.11.0`, `chacha20 0.10.2`, and `aead-stream 0.6.0`. A +one-round E2EE-only run of the benchmark harness measured these +initial-encryption values; the other phases from that round are omitted here. + +| Build | Phase | Time | Peak RSS | Stored logical bytes | Stored allocated size | +|---|---|---:|---:|---:|---:| +| `cargo install --git`, no build flags | Initial encryption | 10.080 s | 1304.61 MiB | 920,636,512 B (877.99 MiB) | 880.02 MiB | + +The row is a single run, not a median. Peak RSS is 1,367,982,080 bytes. Stored +allocated size is 922,767,360 bytes. The no-flags AArch64 build selected the +NEON backend; a stream test with tracing reported +`chacha20_backend=aarch64-neon`. + +### Linux aarch64 cross-check (2026-10-06) + +The same `scripts/reproduce-benchmark.sh` was run from a fresh clone, with an +empty input cache, in a Debian 13 LXC container on an Oracle Cloud +`VM.Standard.A1.Flex` host (Ampere Altra, AArch64), limited to 2 CPUs and +6 GiB RAM, on Btrfs. Git 2.47.3, GnuPG 2.4.7, Rust 1.99.0, e2ee `81e43c1` +(the runner revision; protocol and crypto code identical to the macOS run). +Three rounds, five tiny commits each. `TMPDIR=/var/tmp` was set by hand +because that runner revision still defaulted to the container's tmpfs `/tmp`; +later revisions default to `/var/tmp` on Linux. + +| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial encryption (to a local directory) | 82.7 s | 22.8 s | 24.2 s | +| Initial push | 81.1 s | 82.8 s | 64.1 s | +| Fresh fetch | 92.4 s | 145.7 s | 118.9 s | +| Tiny commit | 0.07 s | 0.07 s | 0.07 s | +| Tiny push | 0.04 s | 23.9 s | 0.22 s | +| Tiny update (fetch) | 0.03 s | 43.4 s | 0.20 s | +| Data sent per tiny push | 4.5 KB | 921 MB | 8.9 KB | +| Peak memory, initial push | 1.11 GiB | 0.96 GiB | 1.11 GiB | +| Peak memory, tiny push | 44 MiB | 884 MiB | 48 MiB | +| Remote size after 5 tiny pushes | 891 MiB | 5,272 MiB | 878 MiB | +| Total client disk after fresh fetch (.git + auxiliary cache/state) | 891 MiB | 1,780 MiB | 1,779 MiB | + +The ordering matches the macOS run except initial encryption, where gcrypt is +about 6% faster on this two-core machine. The client disk total was computed +from the run's `medians.tsv` (client `.git` plus carrier cache), because that +runner revision's summary omitted the cache. + +In that run the first tiny push after the initial push took 9-32 s and about +450 MiB, because the client re-walked the full history before the push; later +pushes took about 0.2 s. After the fix in `f4526d8`, a one-round e2ee-only +rerun on the same container measured 0.15-0.16 s and 6-7 MiB for all three +tiny pushes, including the first. + +### Carrier receiver attribute sensitivity probe (2026-10-02) + +I repeated the E2EE phases once with `BENCH_ROUNDS=1 BENCH_E2EE_ONLY=1 +BENCH_CARRIER_ATTR_TREE=0 BENCH_TRACE=1 BENCH_TINY_COMMITS=5`. This leaves the +local bare carrier receiver's `attr.tree` unset while keeping +`receive.unpackLimit=0`; plain Git and gcrypt were skipped. The probe used the +same Godot and gcrypt revisions above and helper revision +`338d94ef59b8141309355007678e0e0d94a1cbf8`. Its single-run readings are +compared below with the three-round medians above, so small differences include +normal run-to-run noise. + +| E2EE phase | `attr.tree` set: 3-round median | `attr.tree` unset: one run | +|---|---:|---:| +| Initial encryption: wall time; stored bytes | 14.200 s; 920,636,512 B | 15.160 s; 920,636,512 B | +| Initial carrier push: wall time; remote bytes added | 29.460 s; 920,777,562 B | 31.080 s; 920,777,562 B | +| Fresh fetch: wall time; matching remote object bytes | 40.430 s; 920,764,176 B | 41.540 s; 920,764,396 B | +| Tiny commit: median wall time across five commits | 0.160 s | 0.150 s | +| Tiny push: median wall time; median remote bytes per push | 1.230 s; 9,460 B | 1.290 s; 9,460 B | +| Tiny update: median fetch; matching remote bytes per update | 0.890 s; 9,460 B | 0.840 s; 9,460 B | + +The receiver without `attr.tree` did not make fresh fetch materially slower: +41.54 s versus 40.43 s, a 1.11 s difference in these differently sized +samples. The traced carrier-cache fetch from the remote took 6.72 s; object +reads took 0.40 s for 920.6 MB, decryption took 15.75 s, `index-pack` took +28.34 s, and inner-ref connectivity took 3.15 s. The decryption and +`index-pack` timers overlap. Warm update fetches took 0.80--0.94 s, with remote +cache refresh around 0.08 s and incremental decrypt/import around 0.01--0.02 s. + +| Tiny operation | `attr.tree` set: earlier median | `attr.tree` unset: one run | +|---|---:|---:| +| Push 1 | 4.380 s / 9,344 B | 4.160 s / 9,346 B | +| Push 2 | 1.090 s / 9,407 B | 1.170 s / 9,405 B | +| Push 3 | 1.150 s / 9,460 B | 1.290 s / 9,460 B | +| Push 4 | 1.220 s / 9,520 B | 1.260 s / 9,518 B | +| Push 5 | 1.250 s / 9,578 B | 1.370 s / 9,577 B | +| Update fetches 1--5 | 0.730 / 0.790 / 0.910 / 0.890 / 1.050 s | 0.800 / 0.840 / 0.830 / 0.920 / 0.940 s | + +No client-side change was warranted by this result. The test client created +the uploaded carrier pack with delta search disabled, and the receiver kept +incoming packs intact with `receive.unpackLimit=0`. The 6.72 s remote fetch +shows no sign of the expensive server-side repack seen in the pre-optimization +profile; pack reuse is the likely explanation, though this run did not +instrument the server's `upload-pack` internals. A Git client cannot set a +hosted server's `upload-pack` configuration; hosted services may choose +different pack reuse and generation behavior. To reproduce the probe, the +harness accepts `BENCH_E2EE_ONLY=1 BENCH_CARRIER_ATTR_TREE=0`. + +### Historical phase profile before and after carrier optimizations + +These traced single-run figures predate the initial-encryption crypto +acceleration above. The three-round medians earlier in this section are the +current benchmark results. Pack decryption and `index-pack` timers overlap +because Git consumes the decrypted stream while it is being produced; their +durations must not be summed. + +| Operation | Before: wall and main phases | After: wall and main phases | +|---|---|---| +| Initial carrier push | 181.7 s total; cache remote fetch was small; inner pack 14.5 s; `git add e2ee` 22.7 s; carrier `git push` 143.6 s | 32.1 s total; cache fetch 0.05 s, checkout setup 0.15 s; inner pack/encryption 14.7 s; `git add` 10.3 s; commit 0.17 s; carrier push 5.9 s | +| Fresh E2EE fetch | 178.3 s total; carrier cache fetch 143.1 s; object reads 0.08 s / 920.6 MB; decrypt 15.5 s; `index-pack` 28.2 s; connectivity 3.1 s | 44.0 s total; carrier cache fetch 9.3 s; checkout setup 0.18 s; object reads 0.41 s / 920.6 MB; decrypt 15.8 s; `index-pack` 28.3 s; connectivity 3.1 s | +| First tiny push | 141.0 s total; cache remote fetch 133.9 s; checkout 1.14 s; connectivity 3.55 s; `git add` 1.79 s | 4.84 s total; cache refresh and checkout 0.28 s; connectivity 3.78 s; incremental pack 0.02 s; staging 0.01 s | +| Warm tiny push | 3.03 s total; checkout 1.16 s; refresh 0.12 s; connectivity 0.05 s; `git add` 1.08 s | 1.25 s total; checkout 0.17 s; refresh 0.11 s; connectivity 0.11 s; staging 0.01 s | +| Tiny update fetch | 1.50 s total; cache refresh about 0.14 s; checkout about 1.08 s | 0.89 s median; cache fetch 0.09 s; checkout 0.18 s; chain verification 0.13 s; connectivity 0.06 s; decrypt/import about 0.02 s | + +The most effective changes were compression-free carrier Git settings and a +committed `-delta` attribute honored by the local receiver, direct writes of +published objects into the locked cache, and a temporary checkout with an +index but no materialized historical files. Carrier push fell from 143.6 s in +the profile to 5.9 s, cache fetch from 143.1 s to 9.3 s, and the first tiny +push from 141.0 s to 4.84 s. The code keeps a disposable Git directory and +index per helper process rather than sharing a mutable index in the cache. +This avoids cross-process index races while removing the history checkout and +full-tree `git add` scan. Trace can be enabled with +`GIT_REMOTE_E2EE_TRACE=1`; the benchmark passes it and prints per-phase lines +when run with `BENCH_TRACE=1`. + +## Earlier local performance benchmarks + +These earlier measurements use the local filesystem backend or other +backend-specific scenarios. They are retained as historical comparisons and +are not a substitute for the Git-backend results above. The benchmark is opt-in and never runs in CI. It measures the release binaries against a real Git repository using an isolated filesystem backend: @@ -7,6 +368,33 @@ against a real Git repository using an isolated filesystem backend: scripts/benchmark-local.sh /path/to/source/repository ``` +To compare the filesystem backend with a local bare Git remote, run: + +```console +scripts/benchmark-comparison.sh /path/to/source/repository +``` + +The repository also includes two opt-in competitor harnesses. They require +separate local checkouts/builds of the named projects and do not install or +modify those tools globally: + +```console +scripts/benchmark-gcrypt.sh /path/to/source/repository /path/to/git-remote-gcrypt +scripts/benchmark-file-filters.sh /path/to/git-crypt /path/to/transcrypt +``` + +The comparison benchmark sends the exact same commit sequence to both remotes: +an initial push, a tiny change, an incompressible 10 MiB file, a one-byte +modification of that file, and 1,000 unique small files. After the initial +transfer it adds a second E2EE reader, proves +that every existing pack is byte-for-byte unchanged, and fetches the complete +history with only the new reader's private key. Later updates are also fetched +by that new reader. Each push is followed by a fetch into a returning client. +The script records wall time, peak RSS, and both logical and allocated +remote/client disk growth. Git automatic maintenance is disabled so +asynchronous commit-graph or repack work cannot move the disk measurements +after a phase completes. + The script creates a transport clone through `file://` with `--no-local`, removes its `origin`, and keeps every key, encrypted remote, and reconstructed repository in a temporary directory. It never checks out the source's files and @@ -68,3 +456,260 @@ grows with reachable history even when the encrypted delta is tiny. `phases.tsv` records phase name, wall seconds, peak RSS bytes, allocated storage bytes, and logical storage bytes. `summary.json` records aggregate environment, source, pack-count, and storage-size metadata. + +## Single-run comparison with large public repositories + +On 2026-08-16 the comparison benchmark was run once per repository on the same +Apple Silicon Mac, using Rust 1.95.0, the release profile, and crate commit +`f7ef09c`. Only each default branch was cloned, without tags. These are +deliberately single-run exploratory results, not stable thresholds or medians: + +| Repository | Source HEAD | Commits | HEAD files | Reachable data | +|---|---|---:|---:|---:| +| `godotengine/godot` | `00932449c9f3` | 85,678 | 14,162 | 867 MiB | +| `rust-lang/rust` | `67854e511de2` | 336,609 | 62,013 | 926 MiB | +| `kubernetes/kubernetes` | `a231bf3f3776` | 140,375 | 31,300 | 1.16 GiB | + +Initial transfers show that streaming encryption does not add another +whole-pack memory copy. Initial push RSS is effectively the same as Git's and +is dominated by `pack-objects`; E2EE fresh-fetch RSS is lower in all three +runs. The E2EE push does less receiver-side Git processing because the +untrusted backend stores opaque objects rather than indexing plaintext Git +objects, so its local-filesystem time is not a network-server throughput claim. + +| Repository | Initial push Git / E2EE | Push RSS Git / E2EE | Remote write Git / E2EE | Fresh fetch Git / E2EE | Fetch RSS Git / E2EE | +|---|---:|---:|---:|---:|---:| +| Godot | 30.51 / 13.78 s | 1,301 / 1,331 MiB | 890 / 877 MiB | 30.26 / 31.75 s | 1,321 / 601 MiB | +| Rust | 75.77 / 34.28 s | 2,058 / 2,050 MiB | 1,031 / 970 MiB | 73.70 / 63.19 s | 2,064 / 1,429 MiB | +| Kubernetes | 45.46 / 22.38 s | 1,770 / 1,771 MiB | 1,234 / 1,203 MiB | 46.76 / 50.21 s | 1,771 / 1,053 MiB | + +The initial reconstructed client's allocated disk was effectively equal for +Godot (904 MiB each), 1,034 MiB for Git versus 1,082 MiB for E2EE on Rust, and +1,235 MiB versus 1,266 MiB on Kubernetes. Those 0--4.6% differences come from +pack/index layout rather than an additional plaintext checkout; both clients +are non-bare Git repositories with empty working trees. + +Adding a second reader was independent of repository history size in these +runs. The policy transition generated a new key and two recipient envelopes, +but did not create or rewrite a content pack: + +| Repository | `device-add` time | Peak RSS | Remote logical growth | New-reader full fetch | Fetch peak RSS | New client disk | +|---|---:|---:|---:|---:|---:|---:| +| Godot | 0.03 s | 2.73 MiB | 3,379 bytes | 31.73 s | 577 MiB | 904 MiB | +| Rust | 0.03 s | 2.73 MiB | 3,379 bytes | 62.75 s | 1,439 MiB | 1,082 MiB | +| Kubernetes | 0.03 s | 2.75 MiB | 3,379 bytes | 47.01 s | 1,052 MiB | 1,267 MiB | + +Before and after `device-add`, the benchmark compares a sorted list containing +every pack path and SHA-256 digest. All three repositories matched exactly, so +roughly 0.9--1.2 GiB of prior ciphertext remained byte-for-byte unchanged. The +new reader nevertheless reconstructed the complete history using only its own +private key. Its full-fetch cost is therefore similar to any other fresh fetch +and scales with content/history, while the administrative operation itself does +not. + +With two active readers, later manifests were about 304--305 bytes larger than +the corresponding single-reader run. This is the expected per-generation +recipient-envelope cost: metadata grows linearly with active readers, while +pack storage remains independent of reader count. + +### Scaling to 100 active readers + +The Godot workload was repeated with one administrator and 99 collaborators, +added through 99 sequential policy transitions. `BENCH_TOTAL_READERS=100` +enables this mode. Every pre-existing pack path and SHA-256 digest still matched +after all additions, and the 100th reader fetched the complete history and all +later updates using only its own private key. + +The result is bounded and practical at 100 readers, but it is not literally +constant. The immutable content is independent of reader count; policy and +recipient-envelope work is not: + +| Operation | 2 active readers | 100 active readers | +|---|---:|---:| +| Last `device-add` | 0.03 s, 3,379 bytes | 0.12 s, 82,827 bytes | +| New reader full fetch | 31.73 s, 577 MiB RSS | 37.45 s, 575 MiB RSS | +| Returning tiny fetch | 3.24 s, 526 MiB RSS | 3.50 s, 528 MiB RSS | +| New client allocated disk | 904 MiB | 904 MiB | +| Existing pack rewrite | 0 bytes | 0 bytes | + +Adding readers 2 through 100 took 6.42 seconds in total and retained 4.07 MiB +of logical policy/manifest metadata (4.45 MiB allocated). The 100th individual +transition was still small, but the history of adding readers one at a time is +quadratic in aggregate metadata: each immutable policy generation repeats a +larger device list and each manifest repeats a larger envelope list. This does +not multiply content packs. + +Push timing also depends on the client path. The direct diagnostic +`git-e2ee push` has no clone-local continuity pin, so it deliberately validates +the complete 100-generation chain on every push. It took 3.12--3.54 seconds in +this run. Normal Git usage goes through `git-remote-e2ee`; set +`BENCH_E2EE_PUSH_MODE=native` to benchmark that path. Its first push after 99 +unobserved membership changes spent 3.25 seconds catching its continuity pin +up once. Later steady-state pushes were: + +| Update | 2 readers, direct CLI | 100 readers, native Git | +|---|---:|---:| +| Add incompressible 10 MiB | 0.40 s | 0.65 s | +| Add 1,000 small files | 0.34 s | 0.60 s | + +At 100 readers each later manifest added about 32.5 KiB, roughly 29 KiB more +than the two-reader run. Peak push RSS stayed small (about 17--29 MiB after the +one-time catch-up), and remote growth remained the new Git pack plus that small +manifest. Thus reader count does not affect bulk encryption, pack storage, or +client repository size, while HPKE envelope creation adds about 0.25 seconds +and tens of KiB per generation at 100 readers on this machine. + +Incremental push time stayed below one second for every case. Logical remote +growth was approximately 1.4--1.6 KiB for Git versus 3.1--3.3 KiB for E2EE for +the tiny commit, 10,244.5--10,244.7 KiB versus 10,246.4--10,246.5 KiB for the +10 MiB file, and about 110.2 KiB versus 79.3 KiB for the 1,000-file commit. +Pack compression and framing differ, so the smaller side is workload-dependent; +the useful result is that growth remains proportional to new objects rather +than repository size. + +| Repository | Tiny push Git / E2EE | 10 MiB push Git / E2EE | 1,000-file push Git / E2EE | +|---|---:|---:|---:| +| Godot | 0.17 / 0.08 s | 0.60 / 0.40 s | 0.56 / 0.34 s | +| Rust | 0.37 / 0.08 s | 0.62 / 0.40 s | 0.59 / 0.36 s | +| Kubernetes | 0.25 / 0.08 s | 0.58 / 0.39 s | 0.50 / 0.34 s | + +Before verified-frontier connectivity was implemented, the clear bottleneck +was returning-client fetch. E2EE performed a full connectivity walk after +importing even a tiny encrypted delta, so the difference grew with history +size: + +| Repository | Tiny returning fetch Git / E2EE | E2EE peak RSS | +|---|---:|---:| +| Godot | 0.08 / 3.24 s | 526 MiB | +| Rust | 0.35 / 19.65 s | 1,436 MiB | +| Kubernetes | 0.26 / 6.62 s | 1,053 MiB | + +Adding 10 MiB or 1,000 files barely changed those old E2EE returning-fetch +times. This identified history traversal, rather than AEAD throughput or delta +size, as the dominant incremental-read cost. + +The implementation now persists the exact ref tips that passed connectivity +verification. A returning fetch requires those frontier tips to remain local +and asks `rev-list` to inspect only objects newly reachable beyond them. A +missing or legacy frontier still triggers a full walk, and the signed-ref +without-pack attack test now exercises and passes through the incremental path. + +Godot was rerun once with the optimization and the additional one-byte-change +scenario. Initial/fresh operations remained unchanged within single-run noise: +initial E2EE push was 14.32 seconds, fresh fetch was 31.86 seconds, and a new +reader's full fetch was 31.70 seconds. Returning fetch changed substantially: + +| Godot returning update | Plain Git | E2EE before | E2EE verified frontier | +|---|---:|---:|---:| +| Tiny commit | 0.12 s | 3.24 s | 0.09 s | +| Add incompressible 10 MiB | 0.58 s | about 3.2 s | 0.23 s | +| Change one byte in 10 MiB | 0.37 s | not measured | 0.23 s | +| Add 1,000 small files | 0.08 s | about 3.2 s | 0.10 s | + +Peak E2EE RSS for those returning fetches was 39--40 MiB, down from 526 MiB +for the old Godot tiny-fetch path. The reconstructed clients still resolved to +the expected final ref and passed `git fsck --full`; the encrypted remote also +passed full authenticated-object verification. Rust and Kubernetes have not +yet been rerun with this optimization, so their old values above remain useful +as the before baseline rather than a claim about current performance. + +All six reconstructed clients (Git and E2EE for each repository) resolved to +the expected synthetic final commit and passed `git fsck --full`; every E2EE +remote also passed full authenticated-object verification. Rust's upstream +history produced two pre-existing `badFilemode` warnings in both reconstructed +clients, without verification failure. + +Remote logical growth is a useful approximation for local-backend write volume, +not a direct network-byte measurement. Filesystem allocation is also recorded +in the raw TSV. Results can be affected by OS page cache and local filesystem +behavior; repeat at least three times and report medians before making release +claims. + +An earlier pass exposed raw Git's detached post-receive automatic maintenance: +remote metadata could grow between two otherwise read-only measurement phases. +The results above are the clean rerun after disabling `receive.autogc` and +automatic client maintenance. The tables report operation time and immediate +per-push logical delta so phase boundaries remain explicit. + +## Earlier comparison with the tools in the feature table + +The tools in the README do not all encrypt at the same layer, so one combined +ranking would be misleading. The following are two separate, single-run local +workloads on the same Apple Silicon Mac on 2026-08-16. They are exploratory +measurements, not release thresholds. OS page cache and process startup affect +the results; repeat the harnesses and report medians for stronger claims. + +Tool revisions were `git-remote-gcrypt` `a5ff704d071f`, `git-crypt` +`8c7a90ff38fc`, `transcrypt` `1b59c8e505c0`, and `git-remote-e2ee` +`f7ef09c` plus the uncommitted benchmark harnesses described here. + +### Earlier whole encrypted remote: Godot (gcrypt local-filesystem backend) + +This is the direct comparison. Each whole-remote tool received the same Godot +default-branch history (85,678 commits, 14,162 HEAD files, 867 MiB reachable +data), followed by the same synthetic commits. `git-remote-gcrypt` used its +local-filesystem backend, which its own documentation identifies as an +efficient backend. It does **not** represent the Git backend measured above; +these older numbers cannot establish what gcrypt sends to a Git-hosted remote. + +| Metric | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` | +|---|---:|---:|---:| +| Initial push | 30.51 s | 12.26 s | 13.78 s | +| Initial push peak RSS | 1,301 MiB | 1,006 MiB | 1,331 MiB | +| Initial remote write | 890 MiB | 877 MiB | 877 MiB | +| Fresh fetch | 30.26 s | 30.78 s | 31.75 s | +| Fresh fetch peak RSS | 1,321 MiB | 581 MiB | 601 MiB | +| Tiny push | 0.17 s | 0.54 s | 0.08 s | +| Incompressible 10 MiB push | 0.60 s | 0.66 s | 0.40 s | +| 1,000-small-file push | 0.56 s | 0.69 s | 0.34 s | + +The result is narrower than a feature-only comparison might suggest: +gcrypt's local backend is essentially tied with E2EE for initial transfer. +E2EE is faster for these incremental pushes. Before verified-frontier +connectivity, gcrypt's returning fetches were also much faster. After the +optimization, the same update classes took 0.50/0.64/0.47 seconds in gcrypt +versus 0.09/0.23/0.10 seconds in E2EE for tiny/10 MiB/1,000-file fetches. +These are separate single runs and not stable percentage claims, but the old +full-history connectivity bottleneck is no longer present on this workload. + +The gcrypt remote grew by about 1.7 KiB, 10.01 MiB, and 67.2 KiB for the three +updates. E2EE grew by about 3.2 KiB, 10.01 MiB, and 79.3 KiB. Both local +backends therefore stored incremental data rather than rewriting the 877 MiB +history. Backend choice matters: these gcrypt numbers must not be generalized +to its Git-hosted transport. + +### Selected encrypted files: filter microbenchmark + +`git-crypt` and `transcrypt` leave the repository itself visible and encrypt +only matching blobs during `git add`. Running them on every object in the +Godot history would measure a different product contract, so the harness uses +a fresh ordinary Git repository with one selected 10 MiB random file and +1,000 selected 1 KiB random files. It measures filter staging plus a push to a +local bare Git remote. A fresh clone is then unlocked and byte-compared with +the plaintext inputs. The committed blobs are also checked to differ from the +plaintext blob ID, and both clones pass `git fsck --full`. + +| Operation | `git-crypt` | `transcrypt` | +|---|---:|---:| +| Add 10 MiB: stage + push | 1.14 s | 1.76 s | +| Add 10 MiB: remote logical growth | 10.00 MiB | 10.52 MiB | +| Change one byte in that file: stage + push | 1.27 s | 2.00 s | +| One-byte change: remote logical growth | 10.00 MiB | 10.52 MiB | +| Add 1,000 small files: stage + push | 20.03 s | 143.94 s | +| 1,000 files: remote logical growth | 1.06 MiB | 1.12 MiB | +| Fresh clone + unlock | 29.64 s | 380.18 s | +| Peak RSS across measured filter phases | 74.8 MiB | 92.0 MiB | + +For large individual files the filter cost is modest. File count is the more +important result: clean/smudge process startup dominates 1,000 small files, +especially for transcrypt. Filter tools also destroy plaintext similarity +before Git stores the selected blob: in this un-repacked local remote, a +one-byte plaintext change added another complete ciphertext-sized object. +That behavior is inherent to per-file ciphertext, although later Git +maintenance and filesystem compression can change allocated-disk results. + +These numbers should not be read as saying that a whole-remote helper is +universally faster. The filter tools preserve host-side diffs and normal Git +features for everything not selected, use little memory, and avoid encrypting +unselected history. Conversely, their shared-secret and metadata-hiding models +are not substitutes for E2EE's per-device whole-repository design. diff --git a/docs/design.md b/docs/design.md index 2e493ad..80d9731 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,6 +1,6 @@ # Design -This document summarizes the implemented v4 architecture. [`spec.md`](spec.md) +This document summarizes the implemented v5 architecture. [`spec.md`](spec.md) defines the protocol invariants and security claims in detail. ## Goal @@ -35,8 +35,7 @@ private key of a device that remains active. ```text objects/ encrypted incremental Git pack -manifests/ signed header + encrypted generation delta -policies/ signed device registry and roles +manifests/ plaintext discovery header + encrypted sealed header and body HEAD opaque newest-manifest pointer ``` @@ -44,9 +43,10 @@ Only `HEAD` is mutable. Updating it requires compare-and-swap against the exact value observed by the publisher. The manifest body contains complete current refs, only packs introduced by its -generation, and the predecessor-key link. Historical pack descriptors are not -copied into every new manifest. The signed manifest chain is the append-only -pack delta log. +generation, the predecessor-key link, and the complete signed policy when that +manifest introduces one. Historical pack descriptors are not copied into +every new manifest. The signed manifest chain is the append-only pack delta +log; there are no separate `policies/` objects. ## Cryptography @@ -57,11 +57,22 @@ pack delta log. for Git packs - SHA-256 content IDs and generation-key commitments -The signature covers exact header bytes and the encrypted body digest. The -header includes the complete reader-envelope list and generation-key -commitment. Every verifier checks that envelope device IDs exactly equal the -selected policy's active-reader set. Each recipient checks its unwrapped key -against the signed commitment before body decryption. +The plaintext header contains only format version, repository root, generation, +previous manifest ID, generation-key commitment, anonymous padded envelopes, +and the digests of the encrypted body and sealed-header content. The +K_t-encrypted sealed header contains policy ID and generation, signer device +ID, transition type, cumulative pack count, envelope audit, and Ed25519 +signature. The signature covers the exact plaintext header bytes, the sealed +header content without the signature, and the encrypted body digest. + +Each envelope hides its recipient ID. Publishers pad the envelope list to the +next power of two with a minimum of four, sort by encapsulated-key bytes, and +make each dummy a genuine HPKE seal of a random key to a throwaway X25519 +recipient, so its encapsulated key is a real curve point. A device trial +decrypts only the head list; predecessor links provide older keys. Every +verifier uses each audited seed with `rand_chacha` 0.9.0 `ChaCha20Rng` to +re-seal K_t and check byte equality, then requires exactly one real envelope +for each active reader in the encrypted policy. Subkey contexts use the repository root, fixed-width generation, object kind, and dense generation-local ordinal. Pack AAD binds the same identity. Keys are @@ -77,13 +88,15 @@ roles are independent, with two liveness constraints: Readers receive generation envelopes. Writers sign ordinary manifest updates. Administrators sign direct child policies and policy-transition manifests. +The signed policy bytes travel only in the encrypted body of the manifest that +introduces them. Later manifests name the current policy in their sealed header. The repository root is derived from the genesis owner's two public keys. Genesis contains exactly that active owner with reader, writer, and administrator roles and is self-signed. Child policies are authorized by an administrator in the parent policy. -The format retains an administrator threshold and signature array, but v4 +The format retains an administrator threshold and signature array, but v5 accepts threshold 1 and one signature only. Unsupported thresholds fail closed. ## Device operations @@ -105,15 +118,38 @@ administrative CAS is never automatically rebased. - New data is generated with `git pack-objects --stdout --revs`. - Pack output is encrypted as 1 MiB authenticated segments directly into a backend-owned stage; it is never collected into a whole-pack Rust buffer. +- Pack ciphertext hashing uses `sha2`'s runtime-selected hardware assembly + when available, with its portable fallback otherwise. The ChaCha20 backend + selects NEON automatically on supported AArch64 targets, including for + `cargo install --git` builds. The STREAM implementation is provided by + RustCrypto's `aead-stream` crate; switching to it preserves the BE32 + construction. These code-generation choices do not change the stream format + or affect Git configuration in the user's inner repository. + `GIT_REMOTE_E2EE_TRACE=1` reports the compiled ChaCha20 backend. - Existing remote tips present locally are pack exclusions. - Packs are decrypted segment by segment directly into `git index-pack` oldest-first. - After import, every advertised ref must resolve to a complete local Git - object graph before pins or remote-tracking refs move. -- Client-side `git merge-base --is-ancestor` enforces fast-forward updates. -- Explicit force pushes append a normal encrypted delta. -- Push destinations are limited to `refs/heads/*`; tags and deletion fail - before publication. + object graph before the continuity floor or remote-tracking refs move. +- Authentication alone never advances the continuity floor. The floor advances + after a successful import and connectivity check, after the client's own + publication wins compare-and-swap, or during listing when every advertised + ref already resolves to a complete local object graph. The last case matters + because Git skips the helper's fetch command when nothing needs to be + transferred, as after a membership change, a deletion, or a tag pointing at + an existing object; without it, such a state would never be pinned and a + later replay of the previous state would be accepted. +- Per-repository/remote client-state updates hold an advisory file lock across + validation and read-modify-write. The state replacement remains durable and + atomic, and its manifest and policy generations never move backward. +- The helper advertises `refs/heads/*` and `refs/tags/*`; every other ref + namespace is rejected before publication. +- Branch updates require fast-forward ancestry unless force is explicit. Tags + may be created freely, but an existing tag is moved only by a forced push. + Lightweight and annotated tag objects are included in the encrypted pack. +- Branch and tag deletion publish a signed generation with the complete ref set + minus the deleted ref and no pack delta. `refs/heads/main` cannot be deleted + while it is advertised as the remote `HEAD`. A fresh clone traverses the manifest/key chain to genesis. A returning client traverses to its pinned manifest and imports only unseen pack IDs. The client @@ -142,8 +178,9 @@ published. A flush error fails the call. Immutable objects are durable before the pointer moves, within the limits in [`durability.md`](durability.md). Abandoned `.stage-*` files are unreachable; automatic cleanup is deferred to future GC. -Manifest and policy objects still use bounded buffered parsing with a 16 MiB -hard limit; large pack objects always use the streaming path. +Manifest objects still use bounded buffered parsing with a 16 MiB hard limit; +large pack objects always use the streaming path. Policy bytes are parsed from +the decrypted manifest body and are not separate storage objects. An S3 backend can use create-if-absent immutable writes and a conditional HEAD write, but requires a provider capability test; the label "S3 compatible" does @@ -157,12 +194,50 @@ into a dense canonical sequence of 32 MiB chunks, on the dedicated branch A normal fast-forward push is the CAS: two candidates from the same parent cannot both win. -The host sees outer commit timing, chunk sizes/counts, update frequency, public -policy/header metadata, and total growth. Inner Git metadata remains encrypted. - -The current carrier implementation clones a temporary checkout for each helper -process. Uploads are incremental, but a persistent partial-clone cache is -needed before multi-gigabyte repositories are practical. +The host sees outer commit timing, chunk sizes/counts, update frequency, padded +reader count, and total growth. Public keys, device IDs, roles, signer identity, +and membership changes are encrypted; object-size patterns can still suggest +that a membership transition occurred. Inner Git metadata remains encrypted. + +Each helper process fetches the carrier branch into a persistent bare object +cache, then creates a disposable Git directory with an alternate pointing at +the cache. It seeds a process-local index from the base commit with `git +read-tree`, without checking out or copying the historical `e2ee/` files. +Carrier object reads stream blobs through `git cat-file --batch`; publication +stages only the new chunk paths and `HEAD` with a NUL-delimited path list, so +Git does not scan the complete carrier tree. The temporary index keeps +concurrent helper processes from sharing mutable checkout state. + +The cache is keyed by the SHA-256 of a normalized remote URL; a file lock +serializes cache updates and publications across helper processes. Automatic +garbage collection is disabled so borrowed objects remain available. During a +publication, Git writes new objects and the commit into the locked cache object +database. Only after the normal fast-forward push wins does the cache's carrier +ref advance to the new commit. A lost CAS can leave unreachable ciphertext in +the cache, but it does not move the ref. The carrier branch is fetched again +immediately before each CAS, so a stale cache cannot authorize a write against +an outdated branch tip. + +Carrier Git repositories touched by the helper set `core.compression=0`, +`core.looseCompression=0`, `pack.compression=0`, `pack.window=0`, +`pack.depth=0`, and `core.bigFileThreshold=1m`. The committed root +`.gitattributes` marks `e2ee/**` as `-delta`. These settings avoid spending CPU +compressing or deltifying ciphertext, which is already encrypted and +incompressible. They are performance settings, not integrity assumptions; +normal object IDs and the signed manifest checks remain in force. A Git server +must consult that attribute tree for its own `upload-pack` delta search to +avoid server-side deltification. The benchmark's local bare receiver sets +`attr.tree=refs/heads/git-remote-e2ee`; hosted servers control this setting and +may have different server-side pack behavior. + +Recovery publishes a fresh cache generation instead of replacing objects that +an in-flight temporary checkout may still borrow. Obsolete generations are +retained so those checkouts remain valid; repeated cache corruption can therefore +consume additional local disk space. + +The cache contains carrier Git objects and metadata only. It never stores keys, +plaintext, or decrypted repository data. Its default location and the +`GIT_REMOTE_E2EE_CACHE_DIR` override are documented in the [user guide](guide.md#git-host-as-storage-carrier-git-backend). ## Concurrency @@ -179,7 +254,23 @@ motivates future garbage collection. Returning clients pin repository root, manifest ID/generation, and policy generation. They reject rollback, non-descendant state, policy rollback, and -root substitution. +root substitution. A signed manifest seen by list is not yet a pin: helper +state advances after all required packs are imported and every advertised ref +passes the complete-object-graph check, or after the client's own publication +successfully wins compare-and-swap. A missing or unusable pack therefore cannot +raise the continuity floor. + +Each local repository/remote pair has one stable lock file beside its client +state. Helpers hold the exclusive lock while checking the current floor and +performing an import or publication update. State writes merge monotonically +and use the existing flushed temporary-file, rename, and parent-directory +sync path. Administrator pins use the same serialized, monotonic state update +rules after initialization or a successful policy transition. + +State files written by earlier builds are not lowered automatically. Their +head pin may have come from listing, but it is indistinguishable from a valid +pin made after the client's own publication; lowering it could weaken rollback +detection. Storage alone still cannot prevent: @@ -193,16 +284,222 @@ Old manifest headers retain device envelopes. Later compromise of a device private key can retroactively expose every generation addressed to that device. There is no forward secrecy for stored history. +## Recovering from an unverifiable HEAD + +Anyone with storage write access can point `HEAD` at something conforming +clients reject: bytes that fail to parse, a manifest signed by a reader-only +device, or a manifest that fails any other check. Clients already refuse such +a state and stop. Recovery lets an honest writer continue afterwards without +trusting anything the rejected state claims. It is an explicit operation, +`git-e2ee recover`; nothing recovers automatically. + +### Principle + +Recovery never derives a base, refs, policy, keys, or pack inventory from the +rejected state. The default base is the client's continuity floor: the newest +state it accepted after a complete connectivity check or its own successful +publication. A fully validated descendant of the floor may be chosen +explicitly. Anything else storage offers, including a rejected manifest's +`previous` pointer, is only a discovery hint and is validated independently. + +The danger is an update this client never saw. If a legitimate state `L` +followed the floor and storage later replaced it, continuing from the floor +would drop `L`, and if `L` revoked a device, would deliver new keys to that +device again. The client cannot rule this out from the floor alone, so +recovery shows the user what it found and never chooses silently. + +### Classifying the head + +| Class | Meaning | Recovery allowed | +|---|---|---| +| valid | authenticated, authorized, continuous with the floor | not needed | +| invalid | demonstrably bad: unparseable, bad signature, signer not authorized under the authenticated policy, broken audit, commitment, or predecessor link | yes | +| unverifiable | this device cannot open any envelope | never: a valid revocation looks exactly like this to the revoked device | +| unsupported | unknown format version or transition type, even if the parser reports it as a parse error | never: an old client must not erase a newer client's state | +| unavailable | objects missing or a storage error, including failure to read a historical object | never | +| discontinuous | authenticated but older than, or diverging from, the floor | never: this is detected rollback or equivocation and is reported as such | + +These restrictions bind `recover` as well; it cannot override them. A signer ID +read from a sealed header is reported as the claimed signer until its signature +verifies against a device record in the authenticated policy. + +### `git-e2ee recover` + +1. **Discover, read-only.** Classify the head. On the carrier backend, walk the + outer commit history back to the floor's manifest within a fixed budget and + list every authenticated manifest that descends from the floor, with its + generation, outer commit, verified signer, and whether candidates conflict. + A replay of an older manifest found in outer history is reported, never + treated as newer. Exhausting the budget is an error, not a choice. +2. **Refuse ambiguity.** If authenticated descendants conflict, report a fork + and stop. +3. **Select explicitly.** The default offer is the newest validated descendant + of the floor, or the floor itself if none exists. Choosing an older base + than one found is a deliberate history-discarding choice and requires a + separate confirmation. On the directory backend, which has no `HEAD` + history, only the floor can be offered, with the confirmation: "Updates + after this floor may be omitted. This may also omit revocations and + disclose subsequently published content to devices excluded by a newer + policy." +4. **Publish.** The device must be an active writer under the base's policy. + Build `R` with `R.previous = base`, `R.generation = base.generation + 1`, + the predecessor link to the base key, refs and policy from the base, and + compare-and-swap from the observed storage token, which is the current outer + tip on the carrier. The verified base and the CAS token are separate values. + On the carrier the new outer commit is a child of the current tip, so the + push is still a fast-forward. Rebuild the `e2ee/` tree from the ciphertext + objects the base chain requires, restored byte for byte from the carrier's + history or the local cache and checked against their content IDs; if any is + missing, fail as unavailable. A lost CAS fails; the operator must start a + new `recover` invocation and review its fresh discovery report. + +Git does not pass a push of unchanged refs to the helper, so a plain `git push` +cannot perform recovery; `recover` is the only entry point. It must never +publish during `list for-push`. + +Fresh clones have no floor. Unless they hold an authenticated invitation +checkpoint, `recover` states that freshness and fork identity are unverified. + +### Possible automatic recovery later + +Automatic recovery is safe only if the client can prove nothing legitimate +came between its floor and the junk. Comparing the parent's inner `HEAD` value +with the floor is not enough: storage writers can append a legitimate `L`, then +a replay of the floor, then junk, and the junk's parent would still match. A +future automatic mode would have to anchor to the exact outer commit at which +this client established its floor, never move that anchor on a replay, and +require the current tip to be a single-parent child of it. + +### Implementation requirements + +- Validate every accepted ancestor under the policy in force at that point of + the chain; never import policy authority from a rejected manifest. +- Bound discovery by outer commits, candidates, bytes read, and cryptographic + operations, deduplicating repeated IDs. +- Record rejected heads as bounded diagnostic evidence next to the client + state, separate from the floor. Do not blacklist unverifiable or unavailable + states. +- Never lower the floor. A successful recovery advances it to `R`. + +### Current implementation + +`git-e2ee recover` accepts `--remote ` or `--storage `, +plus `--key`, `--repo`, and `--remote-name`. The repository defaults to `.` +and the remote name defaults to `origin`; the continuity floor is read from +`.git/git-remote-e2ee//state.json`. Without `--publish`, the command +only reports. Publishing an older offer requires `--discard-newer`. Directory +publication requires `--accept-stale-floor` and prints the warning above. + +Discovery scans the available carrier history and refuses any history that +contains a merge commit, since legitimate publications are always +single-parent and a merge could hide a legitimate state behind its second +parent. It works within +budgets of 2,048 outer commits, 256 distinct heads, 64 MiB of manifest bytes, +and 1,000,000 estimated cryptographic operations. It scans past replays of the +floor so a legitimate descendant hidden behind a replay can still be offered. +An exhausted budget fails closed. A CAS loss is returned to the operator; +there is no automatic retry with an old selection. This implements the +restart-at-discovery requirement as a fresh command invocation, matching the +requirement that a losing recovery fail without retrying blindly. + +Before publication, required manifests and packs are restored into the +disposable carrier checkout, unneeded ciphertext objects are removed from that +checkout's `e2ee/` tree, and all base-chain packs are imported into a temporary +bare Git repository to verify complete ref connectivity. The temporary +repository is deleted when the command exits; the persistent carrier cache +continues to contain ciphertext Git objects only. + +## Anonymous recipients (v5) + +v4 let storage see each device's per-repository public keys, its roles, and +every membership change. The reason was structural: envelopes carried a +recipient device ID so a device could find its own, and verifiers compared +that ID set with a plaintext policy. v5 removes that exposure so storage learns +only the padded reader count, the same class of leak as `gpg --throw-keyids` in +git-remote-gcrypt. + +### What changes + +1. **Anonymous, padded envelopes.** The plaintext header carries a list of + HPKE envelopes with no recipient ID. The list is padded with dummy + envelopes to the next power of two, with a minimum of four. A dummy is a + genuine HPKE seal of a random key to a throwaway X25519 recipient, so it is + indistinguishable from a real envelope without the recipient private key. + Uniformly random bytes would not do: a real encapsulated key is an X25519 + public key whose top bit is always clear. + The list is sorted by encapsulated-key bytes, so position carries no + identity across generations. The HPKE AAD still binds the repository root, + format, generation, `C_t`, and the recipient's device ID. The recipient + knows its own ID, and the AAD is not transmitted. +2. **Trial decryption, once per fetch.** A device tries its HPKE key against + each envelope of the newest manifest until one opens and the result matches + `C_t`. Older generation keys come from the backward link chain as in v4, so + only the head generation costs trial decryption: O(padded reader count) + X25519 operations per fetch, not per generation. The list length is capped + (4096) to bound work on malicious input. If nothing opens, the client + reports that this device is not an active reader of the current generation: + it was revoked, was never added, or its envelope is malformed. +3. **Sealed header.** Everything that identifies devices stays out of the + plaintext header and inside a sealed header, encrypted under a `K_t`-derived + subkey: policy ID and policy generation, signer device ID, transition type, + cumulative pack count, the envelope audit (item 5), and the Ed25519 + signature. The signature covers the plaintext header bytes, the sealed + header content without the signature, and the encrypted body digest. The + plaintext header keeps only the format version, repository root, + generation, previous manifest ID, `C_t`, the envelope list, and the digests + needed to locate and authenticate the sealed parts. +4. **Policy inside the transition manifest.** v5 removes the separate + plaintext `policies/` objects. A policy travels inside the encrypted body of + the manifest that introduces it: genesis, device add, revoke, or rotate. + Later manifests reference it by policy ID inside their sealed header, and + readers reach it through the backward key chain. Storage can no longer tell + a membership change from a content push, except by size. +5. **Envelope audit.** Without visible recipient IDs, verifiers cannot compare + an envelope ID set with the active-reader set, so a malicious writer could + corrupt one reader's envelope unnoticed by everyone else. The sealed header + therefore records, for each envelope, either the recipient device ID and a + fresh 32-byte seed that initializes the HPKE encapsulation RNG, or a dummy marker. After + opening its own envelope, every verifier deterministically re-seals `K_t` + for each real entry with that seed and the recipient's public key from the + policy. It requires byte equality with the stored envelope and exactly one + envelope per active reader. Dummy entries only need to be absent from the + reader mapping. A seed reveals only the shared secret for that envelope, + which yields `K_t`, and every verifier already holds `K_t`. The seed is + never reused. + +### What storage still learns + +The padded envelope count (an upper bound on active readers), object sizes, +upload timing, generation count, total growth, and the opaque repository root. +Device public keys and IDs, roles, signer identity, and membership data are +encrypted, though size patterns can still suggest a transition. + +### Costs and limits + +- Format break: v5 does not read v4 repositories and ships no migration, which + is acceptable for a prototype. +- For four or more active readers, padding adds fewer than 2x envelope metadata; + one to three readers use the four-entry minimum. +- A device that cannot open any envelope cannot tell revocation from + corruption. Continuity pins still reject rollback for returning clients. +- A reader can still see the complete policy after decryption, exactly as in + v4. +- The revocation race window, retroactive exposure on device-key compromise, + and the absence of forward secrecy are unchanged. + ## Cost model -For new encrypted pack bytes `S`, active readers `N`, and a small envelope `G`: +For new encrypted pack bytes `S`, padded reader count `P`, and a small envelope +`G`: ```text -content push delta S + N*G + O(1) -membership change N*G + O(1) +content push delta S + P*G + O(1) +membership change P*G + O(policy size) large content storage sum(pack ciphertext sizes), independent of N ``` -Envelope metadata remains linear in readers per generation. Fresh clone time is -linear in generations until checkpoint compaction exists. The v4 wire format -reserves a checkpoint transition, but current clients reject it as unimplemented. +Envelope metadata remains linear in padded readers per generation. A policy +transition also carries the signed policy history in its encrypted body. Fresh +clone time is linear in generations until checkpoint compaction exists. The v5 +wire format reserves a checkpoint transition, but current clients reject it as +unimplemented. diff --git a/docs/guide.md b/docs/guide.md new file mode 100644 index 0000000..dec6457 --- /dev/null +++ b/docs/guide.md @@ -0,0 +1,277 @@ +# User guide + +This guide covers everyday use. For why the tool exists and how it compares to +alternatives, see the [README](../README.md). For the protocol, see +the [design overview](design.md) and the [specification](spec.md). + +## Install + +The project builds two binaries: + +- `git-e2ee`: key generation, repository setup, device management, and + diagnostics +- `git-remote-e2ee`: the remote helper Git runs automatically for `e2ee::` URLs + +```console +cargo install --git https://github.com/combinatrix-ai/git-remote-e2ee +git-e2ee --help +``` + +On AArch64 (Apple Silicon or 64-bit Arm Linux), the ChaCha20 dependency selects +its NEON backend automatically, including for `cargo install --git` builds. +Other architectures use the backend selected for their target. + +From a checkout, use `cargo install --path .` instead. Both binaries must be on +`PATH` for Git to find the helper. + +## Keys + +`git-e2ee keygen` writes a private device key file. The first key created for a +repository is its administrator. The key path is stored only in local Git +configuration. The key file is never written to the encrypted remote and must +never be committed. + +The CLI keeps an administrative continuity pin next to an administrator key, +named `.admin-state.json`. Back up that file together with the key. + +## Git host as storage (carrier-Git backend) + +Any ordinary Git repository can carry the ciphertext: GitHub, GitLab, or a bare +repository. It may be empty or contain unrelated branches. `git-remote-e2ee` +uses its own `git-remote-e2ee` branch. + +```console +git-e2ee keygen --output /safe/place/carrier.key.json +git-e2ee carrier-init \ + --remote https://git.example/user/encrypted-carrier.git \ + --key /safe/place/carrier.key.json + +git remote add private \ + 'e2ee::git+https://git.example/user/encrypted-carrier.git' +git config remote.private.e2ee-key /safe/place/carrier.key.json +git push -u private main +``` + +The carrier holds only this outer structure: + +```text +refs/heads/git-remote-e2ee +└── e2ee/ + ├── HEAD + ├── objects/aa//00000000 + ├── manifests/bb//00000000 +``` + +Ciphertext is split into 32 MiB chunks so it can be stored as ordinary Git +blobs. The host can still observe outer commit times, chunk counts and sizes, +update frequency, and total growth. + +Each helper process fetches the carrier branch into a persistent local bare +object cache, then uses a disposable checkout that borrows those cached Git +objects. The cache is shared by helper and `git-e2ee` processes for the same +normalized remote URL. It is stored at +`$XDG_CACHE_HOME/git-remote-e2ee/carrier/` or, when +`XDG_CACHE_HOME` is unset, `~/.cache/git-remote-e2ee/carrier/`. +Set `GIT_REMOTE_E2EE_CACHE_DIR` to override the parent directory that contains +the per-remote cache directories; tests and isolated runs should set it to a +temporary directory. Fetches are serialized across processes, and each refresh +downloads only missing carrier objects. The cache stores carrier Git objects +and metadata only: no keys, plaintext, or decrypted data. + +## Directory as storage (filesystem backend) + +A local or mounted directory can hold the encrypted repository: + +```console +git-e2ee keygen --output /safe/place/repository.key.json +git-e2ee init \ + --storage /srv/encrypted/example \ + --key /safe/place/repository.key.json + +git remote add private 'e2ee::/srv/encrypted/example' +git config remote.private.e2ee-key /safe/place/repository.key.json +git push -u private main +``` + +Do not point two machines at the same directory through a file-sync service. +Concurrent writers rely on the backend's atomic compare-and-swap, which a sync +service does not provide across machines. Use the carrier-Git backend instead. + +## Clone + +Supply the key path once: + +```console +git -c e2ee.key=/safe/place/carrier.key.json clone \ + 'e2ee::git+https://git.example/user/encrypted-carrier.git' +``` + +The helper saves the path as `remote.origin.e2ee-key` in the new clone. Later +`git fetch`, `git pull`, and `git push` need nothing extra. + +## Add a second device + +Each device has its own private key, and only the small public device file ever +leaves it. If machine A created the repository, set up machine B like this. +Substitute the repository root printed by A's `keygen`. + +```console +# Machine B +git-e2ee keygen \ + --repository-root \ + --output /safe/place/machine-b.key.json +git-e2ee device-export \ + --key /safe/place/machine-b.key.json \ + --output machine-b.public.json + +# Machine A, after receiving only machine-b.public.json +git-e2ee device-add \ + --remote https://git.example/user/encrypted-carrier.git \ + --key /safe/place/carrier.key.json \ + --device machine-b.public.json +``` + +Machine B can then clone with its own key, as shown above. + +`device-add --role` grants one of three roles: `read` can clone, fetch, and +pull; `write` adds push access; and `admin` also allows device membership +changes. The default is `write`. For example, add a read-only device with +`--role read`. `device-list` prints the same role names. For the directory +backend, use `--storage ` instead of `--remote ` with +`device-add`, `device-list`, and `device-revoke`. + +## Revoke a device + +`git-e2ee device-list` prints opaque device IDs. Revoke one with: + +```console +git-e2ee device-revoke \ + --remote https://git.example/user/encrypted-carrier.git \ + --key /safe/place/carrier.key.json \ + --device-id +``` + +Adding or revoking a device publishes a fresh generation key in a single +compare-and-swap. The new generation carries one small envelope per active +reader, and all historical manifests and packs stay unchanged. A newly added +device can use the current key's authenticated backward links to decrypt the +complete history. A revoked device keeps what it could already decrypt, but it +receives no key for the new generation or any later one. + +Membership changes need one administrator signature. This is deliberately +**not multisig** yet: the wire format has an administrator threshold and a +signature array so a future version can add M-of-N approval, but this version +refuses any threshold other than 1. + +The administrative CLI does not retry automatically when it loses a +compare-and-swap race. Inspect the winning state and run the command again. + +## Recover from a broken HEAD + +If a storage writer replaces `HEAD` with an invalid manifest, normal fetches +and pushes stop. Run recovery from a Git repository that has already pinned a +known good state for that remote. The default command prints a report and does +not publish or advance the continuity floor: + +```console +git-e2ee recover \ + --remote https://git.example/user/encrypted-carrier.git \ + --key /safe/place/carrier.key.json \ + --repo /work/my-clone \ + --remote-name private +``` + +For a directory backend, use `--storage ` instead of `--remote`. +The local floor is read from +`/.git/git-remote-e2ee//state.json`; use the same repository +and remote name that previously fetched or pushed. A fresh clone has no floor, +so recovery reports that freshness and fork identity are unverified and will +not publish. + +Review the offered manifest IDs, generations, verified signers, replays, and +any fork report. Recovery is allowed only when the current head is demonstrably +invalid. It refuses valid, unverifiable, unsupported, unavailable, and +discontinuous heads. A signer extracted from an invalid head is labelled only +as a claim until its signature is verified. Read-only devices can inspect the +report but cannot publish recovery. + +To publish from the default base, add `--publish`: + +```console +git-e2ee recover \ + --remote https://git.example/user/encrypted-carrier.git \ + --key /safe/place/carrier.key.json \ + --repo /work/my-clone \ + --remote-name private \ + --publish +``` + +The default base is the newest authenticated descendant found after the local +floor, or the floor if there is no descendant. Recovery refuses conflicting +authenticated descendants. Selecting an older offered base requires both +`--base ` and `--discard-newer`; this can omit legitimate changes +or a revocation. A lost compare-and-swap fails. Start a new command and review +the fresh report before trying again. + +The directory backend has no carrier commit history, so it can offer only the +local floor. Publishing requires `--accept-stale-floor` and prints this warning: + +> Updates after this floor may be omitted. This may also omit revocations and +> disclose subsequently published content to devices excluded by a newer +> policy. + +Carrier recovery restores required ciphertext from prior outer commits when +available, checks each content ID, verifies the selected refs' complete Git +object graph in temporary local storage, then publishes a normal fast-forward +carrier commit. If required ciphertext is no longer available, recovery stops. +After successful recovery, run the usual `git fetch` or `git pull` to update +your local remote-tracking refs. + +## Limits today + +- Pushes support branches under `refs/heads/*` and tags under `refs/tags/*`; + other namespaces are rejected. Branches keep fast-forward checks, while an + existing tag needs `--force` to move. Branch and tag deletion publish a new + encrypted generation without a pack. The helper advertises `main` as the + default branch while it exists, so it refuses to delete `refs/heads/main`. +- After deleting a remote ref, use `git fetch --prune --prune-tags` on a + returning clone to remove its stale local remote-tracking branch or tag. +- A push that loses a race with another writer fails. Fetch, integrate, and + push again. +- A new or deleted local carrier cache downloads the complete carrier branch. + Later operations reuse its objects and fetch only new carrier objects. + `GIT_REMOTE_E2EE_CACHE_DIR` overrides the cache parent directory. +- Recovery discovery has fixed limits of 2,048 carrier commits, 256 distinct + heads, 64 MiB of manifest data, and 1,000,000 estimated cryptographic + operations. It fails closed if any limit is exhausted. +- Shallow and partial clones are not supported. A fresh clone downloads and + verifies the complete history. + +## Storage contract + +Backends implement four logical operations. Immutable object writes are staged +so the final ciphertext ID can be learned while streaming: + +```text +begin_object(kind) -> writable stage; stage.finish(id) +open_object(kind, id) -> reader +read_head() +compare_and_swap_head(expected, next) +``` + +The filesystem backend implements head compare-and-swap with an advisory lock +and a rename in the storage root. Object publication hard-links a staged file +from `.staging` into `objects//` on the same filesystem and never +replaces an existing ID. File contents are flushed before a name is published, +new directories are flushed through the preexisting ancestor, and the parent +directory is flushed again afterwards; a flush error fails the call. Crash and +power-loss limits are in [durability.md](durability.md). The carrier-Git +backend implements compare-and-swap as a normal fast-forward push to the outer +branch. A future S3 backend can use multipart upload plus conditional writes, +but each provider must be capability-tested: "S3 compatible" does not by itself +promise correct compare-and-swap behavior. + +A killed filesystem writer can leave an unreachable file under `.staging/`. +Nothing published points to it, and stale `.stage-*` entries may be deleted +when no writer is running. Automatic cleanup belongs to future garbage +collection. diff --git a/docs/spec.md b/docs/spec.md index c24b1f1..e444af2 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1,9 +1,9 @@ # Protocol specification > [!WARNING] -> This document specifies the experimental v4 format implemented by the current -> prototype. The format is incompatible with v2 and v3. There is no automatic -> migration; keep an independent plaintext copy and every device key. +> This document specifies the experimental v5 format implemented by the current +> prototype. The format is incompatible with v2, v3, and v4. There is no +> automatic migration; keep an independent plaintext copy and every device key. ## 1. Requirements @@ -20,9 +20,11 @@ It is designed to satisfy all of the following: key or plaintext. 3. A publication never rewrites or retransmits historical Git packs. 4. Pack ciphertext is stored once, independent of reader count. Only small key - envelopes scale with the active-reader set. + envelopes scale with the padded reader count; a policy transition also + carries its encrypted policy body. 5. A newly admitted reader receives full history by default. -6. Concurrent publications have exactly one visible winner. +6. Concurrent publications have exactly one visible winner, provided storage + honors the compare-and-swap contract. The protocol provides confidentiality, authenticity, integrity, and continuity from state pinned by the same client. It does not provide global freshness or @@ -43,21 +45,33 @@ prevent an authorized reader from exporting plaintext. subkey derived from `K_t`. - **Policy**: the signed device registry and role assignment. - **Manifest**: the signed generation transition, containing an encrypted body. +- **Sealed header**: Kₜ-encrypted manifest metadata, including policy selection, + signer, transition type, pack count, envelope audit, and signature. +- **Padded reader count**: the number of envelopes after padding to the next + power of two, with a minimum of four. - **Repository root**: the stable identity derived from the genesis owner's Ed25519 and HPKE public keys. - **Device ID**: a domain-separated digest of one device's two public keys. ## 3. Cryptographic construction -The v4 format uses: +The v5 format uses: - Ed25519 signatures; - X25519/HKDF-SHA-256/ChaCha20-Poly1305 HPKE Base mode; -- HKDF-SHA-256 for object subkeys; +- HKDF-SHA-256 for object subkeys, including a distinct sealed-header subkey; - XChaCha20-Poly1305 for manifest and predecessor-link encryption; - XChaCha20-Poly1305 STREAM with a big-endian 32-bit counter for packs; - SHA-256 for content IDs and generation-key commitments. +The v5 domain strings are `git-remote-e2ee subkey derivation v5\0`, +`git-remote-e2ee generation key commitment v5\0`, +`git-remote-e2ee manifest v5\0`, and `git-remote-e2ee policy v5\0`. +`sign_domain` signs `domain || SHA-256(exact_bytes)` with Ed25519. Symmetric +XChaCha20-Poly1305 envelopes are encoded as +`E2EES05\0 || nonce_24 || ciphertext`. Subkey kind values are +`manifest-body=1`, `pack=2`, `predecessor-link=3`, and `sealed-header=4`. + Every successful publisher samples `K_t` independently from the operating system CSPRNG. Repository content, commit IDs, previous keys, timestamps, or a forward KDF MUST NOT be used as the entropy source. In particular, a holder of @@ -68,27 +82,38 @@ The publisher derives distinct subkeys using an injective, fixed-width context: ```text HKDF-SHA-256( input_key = K_t, - salt = protocol-specific v4 domain, + salt = protocol-specific v5 domain, info = repository_root || generation_u64_le || kind_u8 || ordinal_u64_le ) ``` -The defined kinds are `manifest-body`, `pack`, and `predecessor-link`. A key is -used for one logical message only. Manifest and predecessor-link envelopes each -carry a fresh random 24-byte nonce. Pack streams carry a fresh random 19-byte -nonce prefix; the STREAM counter and final-segment marker complete the nonce. -Associated data binds the v4 domain, repository root, generation, object kind, -ordinal or parent manifest ID as applicable, and the complete pack-stream -header. +The defined kinds are `manifest-body`, `pack`, `predecessor-link`, and +`sealed-header`. A key is used for one logical message only. Manifest, +predecessor-link, and sealed-header envelopes each carry a fresh random 24-byte +nonce. Pack streams carry a fresh random 19-byte nonce prefix; the STREAM +counter and final-segment marker complete the nonce. Associated data binds the +v5 domain, repository root, generation, object kind, ordinal or parent manifest +ID as applicable, and the complete pack-stream header. Sealed-header AAD is the +exact plaintext header bytes. -The signed header contains the full, untruncated commitment: +The signed manifest header contains the full, untruncated commitment: ```text -C_t = SHA-256("git-remote-e2ee generation key commitment v4" || K_t) +C_t = SHA-256("git-remote-e2ee generation key commitment v5\0" || K_t) ``` -Commitments are compared in constant time before the corresponding key is used -for AEAD decryption. XChaCha20-Poly1305 itself is not treated as key-committing. +Generation envelopes use HPKE Base mode with info +`git-remote-e2ee generation key v5`. Their AAD binds the repository root, +format version, generation, `C_t`, and recipient device ID. The recipient ID +and AAD are not transmitted. AAD encoding is the v5 envelope domain string, +then a little-endian `u32` length and bytes for repository root, format version +as little-endian `u32`, generation as little-endian `u64`, then a length and +bytes for the ASCII `C_t` and ASCII recipient device ID. Each real envelope uses an independent random +32-byte seed to initialize `rand_chacha` 0.9.0 `ChaCha20Rng`; this seeded RNG is +passed to HPKE 0.13 `single_shot_seal`. The seed is recorded only in the sealed +header audit. Commitments are compared in constant time before the corresponding +key is used for AEAD decryption. XChaCha20-Poly1305 itself is not treated as +key-committing. ## 4. Backward key chain @@ -115,7 +140,7 @@ Consequences: compromise of a device private key that remains an active recipient. - A new reader given `K_t` can traverse to genesis and therefore always gets full history. -- Future-only onboarding is not supported by v4 because the predecessor link is +- Future-only onboarding is not supported by v5 because the predecessor link is available to every reader of the current generation. ## 5. Stored objects @@ -125,25 +150,27 @@ stored bytes: ```text objects/ encrypted incremental Git packs -manifests/ signed header plus inline encrypted delta body -policies/ signed plaintext device registry and roles +manifests/ plaintext discovery header plus encrypted sealed header and body HEAD opaque newest-manifest ID ``` Private device keys and decrypted generation keys are never stored remotely or -in client continuity pins. +in client continuity pins. A policy is included in the encrypted body of the +manifest that introduces it; there are no separate `policies/` objects. -Policies and manifest headers are plaintext-structured to avoid a key-discovery -cycle. This intentionally exposes reader count, repository-specific public -keys, roles, policy changes, ciphertext size, and update timing. Inner refs, -Git object IDs, paths, authors, messages, pack contents, generation keys, and -predecessor-link plaintext remain encrypted. +The plaintext manifest header contains only format version, repository root, +generation, previous manifest ID, `C_t`, the anonymous padded envelope list, and +digests for the encrypted body and sealed-header content. Policy data, device +public keys and IDs, roles, signer, authorization type, pack count, audit, and +signature are encrypted. Inner refs, Git object IDs, paths, authors, messages, +pack contents, generation keys, and predecessor-link plaintext remain +encrypted. ## 6. Policy A policy contains: -- format version, repository root, generation, and previous policy ID; +- format version 5, repository root, generation, and previous policy ID; - administrator threshold and signature array; - immutable historical device records with public keys, roles, and optional revocation generation. @@ -157,41 +184,71 @@ Roles are: Every active writer and administrator MUST also be an active reader because a publisher needs the current generation key to construct the next predecessor -link. A policy must retain at least one active reader and administrator. +link. A policy must retain at least one active reader and administrator. The +complete signed policy bytes are carried inside the encrypted manifest body +that introduces them: genesis and each add, revoke, or rotate transition. +Ordinary manifests do not repeat the policy; their sealed header names its +policy ID and generation. Genesis has exactly one active owner with all three roles and is self-signed. Every child policy is signed by an administrator in its direct parent, never -solely by authority introduced in the child. v4 supports threshold 1 and one +solely by authority introduced in the child. v5 supports threshold 1 and one signature; other thresholds fail closed. A future M-of-N format must count distinct parent administrators under the parent's threshold. ## 7. Manifest -### 7.1 Signed plaintext header +### 7.1 Plaintext and sealed headers -The header binds at least: +The plaintext header contains only: -- format version and repository root; -- generation and exact previous manifest ID; -- selected policy ID and policy generation; -- cumulative pack count; +- format version 5, repository root, generation, and exact previous manifest + ID; - `C_t`; -- the complete, sorted generation-envelope list; -- signer device ID and transition type. - -The Ed25519 signature covers the exact stored header bytes and the SHA-256 -digest of the exact encrypted body bytes. Verifiers never sign or verify a +- the generation-envelope list, with no recipient IDs; +- the SHA-256 digest of the encrypted body; +- the SHA-256 digest of the sealed-header content without its signature. + +The envelope list length MUST be a power of two from 4 through 4096. Publishers +wrap `K_t` once for each active reader, add indistinguishable dummy entries +(each a genuine HPKE Base-mode seal of a fresh random 32-byte key, under a +random AAD, to a freshly generated throwaway X25519 recipient; uniformly random +bytes MUST NOT be used because a real encapsulated key is an X25519 public key +with its top bit clear) until the list +reaches the next power of two, with a minimum of four, then sort by raw +encapsulated-key bytes. Encapsulated keys and ciphertexts MUST have their +protocol-defined lengths. Duplicate or non-increasing encapsulated keys are +invalid. If padding would exceed 4096 entries, publication fails. + +A reader tries its private key against each envelope in the current head until +HPKE opens a 32-byte key matching `C_t`. This is the only trial-decryption pass; +older keys come from predecessor links. If no envelope opens to a key matching +`C_t`, the client MUST report that it is not an active reader of the current +generation, which may mean revoked, never added, or a malformed envelope. + +The sealed header is encrypted under a `K_t`-derived `sealed-header` subkey. +Its content contains policy ID and generation, signer device ID, transition +type, cumulative pack count, and one audit entry per envelope. The entry is +either `(recipient device ID, 32-byte encapsulation seed)` or a dummy marker. +The sealed header also contains the Ed25519 signature. Every real entry's seed +MUST be sampled freshly from the operating system CSPRNG and MUST NOT be reused. +Audit entries correspond position-for-position to the sorted envelope list. + +The signature covers the exact plaintext header bytes, the exact sealed-header +content bytes without the signature, and the encrypted body digest. The signed +encoding length-prefixes each value. The encrypted body digest and sealed +content digest MUST match their exact bytes. Verifiers MUST NOT verify a re-serialized interpretation. -The envelope for one recipient uses HPKE AAD that binds the repository root, -format, generation, policy ID, recipient device ID, and `C_t`. HPKE Base mode -does not authenticate the sender by itself; sender authenticity comes from the -manifest signature covering the complete envelope list. - -Every verifier MUST compare the envelope device-ID set with the selected -policy's active-reader set. The relationship is bijective: no omissions, -extras, or duplicates. A recipient unwraps its envelope and checks `C_t` before -decrypting the body. +After opening its own envelope, every verifier MUST validate the complete +sealed audit. For each real entry it initializes `rand_chacha` 0.9.0 +`ChaCha20Rng` from the 32-byte seed, deterministically seals `K_t` to that +device's HPKE public key using the v5 envelope AAD, and requires byte equality +with the corresponding stored envelope. Real audit device IDs MUST equal the +policy's active-reader set exactly: no missing, duplicate, extra, or revoked +readers. Dummy entries MUST have no reader mapping. HPKE Base mode does not +authenticate the sender itself; signer authorization and the signature provide +that property. ### 7.2 Encrypted delta body @@ -199,7 +256,14 @@ The body contains: - complete current inner refs; - only pack descriptors introduced by this generation; -- exactly one predecessor-key link, except at genesis. +- exactly one predecessor-key link, except at genesis; +- the complete signed policy bytes when this manifest introduces a policy. + +Genesis and policy-transition manifests MUST contain exactly one introduced +policy. Ordinary writer manifests MUST contain none. The policy ID in the +sealed header is SHA-256 of the exact policy object bytes. Verifiers MUST check +that the introduced policy matches this ID and validate its signature and +direct-parent relationship before accepting the manifest. A pack descriptor contains ciphertext ID, plaintext size, creation generation, and a dense generation-local ordinal beginning at zero. Its subkey and AAD bind @@ -214,7 +278,7 @@ historical descriptors, avoiding quadratic cumulative-inventory metadata. Each pack is encoded as: ```text -"E2EEPK4\0" || chunk_size_u32_le || nonce_prefix_19 || segments... +"E2EEPK5\0" || chunk_size_u32_le || nonce_prefix_19 || segments... ``` `chunk_size` MUST equal 1,048,576 bytes. Every non-final plaintext segment is @@ -229,8 +293,10 @@ bytes, and verify SHA-256 of the complete stored stream against the descriptor ID. A malformed header, size mismatch, counter overflow, short read, reordered or duplicated segment, authentication failure, trailing byte, or hash mismatch is fatal. Plaintext may stream into `git index-pack` before the final object hash -is known, but no ref or continuity pin may move until all cryptographic, -content-address, process, and Git-connectivity checks succeed. +is known, but no imported-state pin or remote ref may move until all +cryptographic, content-address, process, and Git-connectivity checks succeed. +Listing an authenticated manifest does not advance the continuity floor. A +publisher advances its own floor only after its compare-and-swap succeeds. Immutable writes MUST be staged within the storage backend and become visible under their ciphertext ID only after the complete streamed hash matches that @@ -270,9 +336,12 @@ genesis policy, contains empty refs and no packs, and has no predecessor link. genesis. 2. Enforce Git fast-forward rules locally unless force was explicit. 3. Create only the incremental Git pack needed for the update. -4. Sample `K_t`, derive the pack and body subkeys, and encrypt the new data. +4. Sample `K_t`, derive the pack, body, and sealed-header subkeys, and encrypt + the new data. 5. Encrypt the previous generation key under the predecessor-link subkey. -6. HPKE-wrap `K_t` once to every active reader. +6. HPKE-wrap `K_t` once to every active reader using a fresh audit seed, add + dummy envelopes sealed to throwaway recipients up to the padded count, sort by encapsulated-key bytes, + and put the corresponding audit entries in the sealed header. 7. Upload the immutable pack and manifest. 8. Compare-and-swap `HEAD` from the observed parent ID to the new manifest ID. @@ -303,6 +372,23 @@ Rotation is one administrative transition that adds the replacement device and revokes the old record. Other users keep their device private keys. No pack is rewritten. +### 8.5 Inner Git refs + +The remote helper advertises refs in `refs/heads/*` and `refs/tags/*`. All other +namespaces MUST be rejected before publication. Branch refs MUST resolve to +commits and retain the fast-forward check unless the writer explicitly forces +the update. Lightweight and annotated tags MAY point to any Git object; their +complete object graphs, including annotated tag objects, MUST be included in +the encrypted pack and verified by receiving clients. An existing tag MUST +remain unchanged unless the update is forced. + +Deleting an existing branch or tag publishes a signed writer manifest with the +complete ref map minus that ref. A deletion introduces no pack and does not +increase the cumulative pack count. The helper advertises `refs/heads/main` as +the remote `HEAD` when present and MUST refuse to delete it while it is +advertised as the default branch. Returning clones can remove stale local +tracking refs with Git's `--prune` and `--prune-tags` fetch options. + ## 9. Delta traversal and Git connectivity A fresh clone traverses manifest and key links from `HEAD` to genesis, @@ -319,6 +405,87 @@ refs, pinning the new head, or publishing a successor. A signed ref advance whose required pack delta is absent is invalid even when every cryptographic check succeeds. +Authenticating a manifest alone MUST NOT advance the continuity floor. A +helper client's head, generation, repository root, and policy-generation floor +advances only when the complete-ref connectivity check succeeds for that +manifest's refs, either after pack import or, during listing, because every +advertised ref is already connected locally; or after that client successfully +publishes its own successor with compare-and-swap. A client MUST NOT rely on +receiving a fetch command to pin a state, because Git omits it when no objects +need to be transferred. A failed or incomplete +import leaves the previous floor intact, so an authenticated but unusable +publication cannot make a later valid successor look like rollback. + +Client-state updates for one repository and remote MUST serialize their +read-modify-write operation across helper processes. The merge MUST preserve +the greatest recorded manifest and policy generations and MUST reject a +same-generation head-ID conflict. The durable state-file replacement remains +atomic; the lock is held while an import or publication checks and advances +the floor. + +A client MAY satisfy that requirement incrementally. After a successful full +or incremental connectivity check, it records the exact checked ref tips as a +verified frontier. On a later fetch it first requires every frontier tip to +still exist locally, then walks the current tips while excluding objects +reachable from the verified frontier. Content-addressed Git objects reachable +from that frontier were already checked and cannot be changed in place, so an +inductive check of only the newly reachable range establishes completeness of +the new state. The frontier advances only after pack authentication, import, +and the incremental connectivity check all succeed. + +A missing frontier in a v5 client state requires a full connectivity walk. A +client state from another wire version is not accepted. The incremental rule assumes the trusted +client's existing object database has not been corrupted outside this +protocol; explicit full verification remains available to check local storage. + +### Explicit recovery + +`git-e2ee recover` MUST default to a read-only report. It MUST classify the +observed storage head as valid, invalid, unverifiable, unsupported, unavailable, +or discontinuous. Recovery publication is permitted only for an invalid head. +Unverifiable, unsupported, unavailable, and discontinuous heads MUST never be +overridden. In particular, a device that cannot open the current envelope +MUST NOT recover from an older floor, because the head may be a valid +revocation. A signer ID extracted before successful signature verification +MUST be labelled as a claim, not as the signer. + +On carrier Git, discovery MUST inspect bounded outer history and MUST refuse +to proceed if that history contains any merge commit: every legitimate +publication is a single-parent fast-forward, and a merge can hide a legitimate +state behind a non-first parent. Discovery MUST deduplicate manifest IDs and +validate candidates against the authenticated policy and predecessor chain. The byte, outer-commit, candidate, and +cryptographic-operation budgets MUST be fixed. Exhaustion MUST fail closed. +Discovery MUST report authenticated descendants and replays. If authenticated +descendants conflict, recovery MUST refuse to choose a fork. A replayed older +manifest MUST NOT be treated as a newer candidate. Directory storage has no +outer history and MUST offer only the pinned floor. + +The default base MUST be the newest validated descendant of the floor, or the +floor when no descendant is available. Selecting an older offered base MUST +require an explicit history-discard confirmation. Directory publication MUST +require explicit acknowledgement that updates and revocations after the floor +may be omitted and that subsequently published content may be disclosed to +devices excluded by a newer policy. + +Before publication, the device MUST be an active writer under the selected +base's policy. Recovery MUST publish a normal signed writer successor whose +previous ID and generation are the selected base and its generation plus one, +whose refs and policy are copied from that base, and whose predecessor link +opens to the base key. It MUST compare-and-swap from the observed storage token +while keeping that token separate from the selected manifest base. Required +base-chain ciphertext MUST be available and checked against content IDs. A +carrier publication MUST restore required historical ciphertext byte-for-byte, +publish a child of the observed outer tip, and use a normal fast-forward push. +Before publishing, the selected refs MUST resolve to complete local Git object +graphs. A lost CAS MUST fail without a blind retry. + +Recovery MUST record invalid rejected heads as bounded diagnostic evidence +separate from client continuity state. That evidence MUST NOT lower the floor +or make an unverifiable or unavailable head ineligible for a later valid +operation. A successful recovery advances the floor only after its own CAS +succeeds. Recovery MUST NOT run from remote-helper `list` or `list for-push`, +and an ordinary `git push` MUST NOT trigger it. + Force pushes append a normal pack delta and new ref state. Older packs remain in historical manifests and may become unreachable in the inner Git graph. @@ -326,7 +493,10 @@ in historical manifests and may become unreachable in the inner Git graph. The server does not inspect encrypted inner commit parents. Two writers read the same opaque `HEAD`, create immutable candidates, and attempt the same CAS. -Exactly one wins. +Against storage that honors the CAS contract, exactly one wins. Malicious or +noncompliant storage can report outcomes inconsistently; that is outside the +guarantees, and a noncooperating writer to directory storage can also cause an +ABA change of the compared `HEAD` value. - Filesystem storage uses locking. `HEAD` is published by rename inside the storage root. Immutable objects are hard-linked from `.staging` into @@ -343,9 +513,14 @@ Unreachable losing objects require later GC. ## 11. Continuity and invitations -Returning clients pin at least repository root, manifest ID and generation, -and policy ID and generation. They reject rollback, a non-descendant manifest, -policy rollback/sideways movement, root substitution, and generation gaps. +Returning clients pin repository root, manifest ID and generation, and policy +generation. The policy ID is not stored separately: the pinned manifest ID +binds it through the manifest's sealed header. Helper continuity pins advance +only after the complete-ref connectivity check succeeds (after import, or +during listing when the advertised refs are already connected locally) or +after a successful own publication, never from authentication alone. They reject rollback, a non-descendant manifest, policy +rollback/sideways movement, root substitution, and generation gaps. Local +client-state updates are serialized and monotonic across concurrent helpers. A fresh device should receive an administrator-authorized invitation checkpoint through an authenticated channel containing: @@ -368,18 +543,20 @@ prove global freshness; gossip or transparency anchoring is a separate layer. ## 12. Cost model -Let `N` be active readers, `G` one small HPKE envelope, and `S` newly encrypted -pack bytes. +Let `N` be active readers, `P` the padded envelope count, `G` one small HPKE +envelope, and `S` newly encrypted pack bytes. `P` is the next power of two at +least `max(N, 4)`. | Operation | Upload delta | Remote storage delta | | --- | ---: | ---: | -| Content push | `S + N*G + O(1)` | `S + N*G + O(1)` | -| Add/revoke/rotate | `N*G + O(1)` | `N*G + O(1)` | +| Content push | `S + P*G + O(1)` | `S + P*G + O(1)` | +| Add/revoke/rotate | `P*G + O(policy size)` | `P*G + O(policy size)` | Large-content storage is `sum(pack ciphertext sizes)`, independent of `N`. -Envelope metadata is still linear in readers per generation and accumulates as -approximately `O(generations * readers * G)`. Fresh clone work is linear in -manifest generations until checkpoint compaction exists. +Envelope metadata is linear in the padded reader count per generation and +accumulates as approximately `O(generations * P * G)`. A membership transition +also carries the signed policy history inside its encrypted body. Fresh clone +work is linear in manifest generations until checkpoint compaction exists. ## 13. Security limits @@ -388,22 +565,24 @@ manifest generations until checkpoint compaction exists. pack. It exposes no later generation. - Compromise of an active device private key is stronger: the attacker can unwrap later generation keys while the device remains active. -- Old immutable headers retain envelopes. Later device-key compromise can - retroactively expose every generation addressed to that device. v4 has no +- Old immutable headers retain anonymous envelopes. Later device-key compromise + can retroactively expose every generation addressed to that device. v5 has no forward secrecy for stored history. - Full-history onboarding is mandatory; future-only access requires a new format or an explicit compartment. -- Storage observes sizes, timing, generation count, reader count, public keys, - roles, and membership changes. +- Storage observes padded reader count, object sizes, upload timing, generation + count, total growth, and opaque repository root. It does not directly observe + device public keys or IDs, roles, signer identity, or membership changes, + although size patterns may reveal a membership transition. - Storage can delete, freeze, or deny service. - Writers can author destructive Git history; force remains explicit. - Global rollback/equivocation detection requires an external anchor. ## 14. Migration and future work -v4 is intentionally wire-incompatible with v2 and v3. Migration must occur on -a trusted client able to decrypt the older repository and republish under a new -v4 repository boundary. Automatic migration is not implemented. The local key +v5 is intentionally wire-incompatible with v2, v3, and v4. Migration must occur +on a trusted client able to decrypt the older repository and republish under a +new v5 repository boundary. Automatic migration is not implemented. The local key file remains format 3 because its stable repository/device identity material did not change; manifest, policy, signature, KDF, HPKE, and AEAD domains changed and therefore fail closed across wire versions. @@ -413,6 +592,5 @@ Planned extensions: 1. checkpoint/compaction and garbage collection with explicit authority; 2. M-of-N administrator authorization and recovery; 3. automatic stale-push retry with explicit merge behavior; -4. persistent partial-clone carrier cache; -5. S3 backend with conditional-write compatibility tests; -6. optional gossip or transparency-log anchoring. +4. S3 backend with conditional-write compatibility tests; +5. optional gossip or transparency-log anchoring. diff --git a/scripts/benchmark-comparison.sh b/scripts/benchmark-comparison.sh new file mode 100755 index 0000000..956b157 --- /dev/null +++ b/scripts/benchmark-comparison.sh @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: benchmark-comparison.sh SOURCE_GIT_REPOSITORY [RESULTS_DIRECTORY] + +Compares a local bare Git remote with the git-remote-e2ee filesystem backend. +The source repository is cloned into an isolated temporary directory and is +never modified. Set BENCH_TOTAL_READERS to an integer of at least 2 to exercise +recipient scaling (default: 2). Set BENCH_KEEP_WORK=1 to retain the temporary +directory. Set BENCH_E2EE_PUSH_MODE=native to measure pushes through the Git +remote helper instead of the direct diagnostic CLI (default: direct). +EOF +} + +if [[ $# -lt 1 || $# -gt 2 ]]; then + usage + exit 2 +fi + +source_repo=$1 +total_readers=${BENCH_TOTAL_READERS:-2} +e2ee_push_mode=${BENCH_E2EE_PUSH_MODE:-direct} +if [[ ! $total_readers =~ ^[0-9]+$ ]] || (( total_readers < 2 )); then + echo "BENCH_TOTAL_READERS must be an integer of at least 2" >&2 + exit 2 +fi +if [[ $e2ee_push_mode != direct && $e2ee_push_mode != native ]]; then + echo "BENCH_E2EE_PUSH_MODE must be direct or native" >&2 + exit 2 +fi +project_root=$(cd "$(dirname "$0")/.." && pwd -P) +results_dir=${2:-$project_root/bench-results/comparison-$(date -u +%Y%m%dT%H%M%SZ)} +source_repo=$(cd "$source_repo" && pwd -P) +results_dir=$(mkdir -p "$results_dir" && cd "$results_dir" && pwd -P) + +git -C "$source_repo" rev-parse --verify HEAD >/dev/null +source_ref=$(git -C "$source_repo" symbolic-ref --quiet HEAD) || { + echo "source repository must have a symbolic branch checked out" >&2 + exit 2 +} +source_branch=${source_ref#refs/heads/} +source_head=$(git -C "$source_repo" rev-parse HEAD) +source_git_dir=$(git -C "$source_repo" rev-parse --absolute-git-dir) +source_git_kib=$(du -sk "$source_git_dir" | awk '{print $1}') +available_kib=$(df -Pk "${TMPDIR:-/tmp}" | awk 'NR == 2 {print $4}') +required_kib=$((source_git_kib * 6)) +if (( available_kib < required_kib )); then + echo "insufficient temporary disk: need about $((required_kib / 1024 / 1024)) GiB" >&2 + exit 1 +fi + +case $(uname -s) in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform for RSS measurement: $(uname -s)" >&2; exit 2 ;; +esac +[[ -x /usr/bin/time ]] || { + echo "/usr/bin/time is required" >&2 + exit 1 +} + +if [[ ${BENCH_SKIP_BUILD:-0} != 1 ]]; then + cargo build --release --bins --manifest-path "$project_root/Cargo.toml" +fi +git_e2ee="$project_root/target/release/git-e2ee" +[[ -x $git_e2ee ]] || { + echo "release binary not found: $git_e2ee" >&2 + exit 1 +} + +work_parent=${TMPDIR:-/tmp} +work=$(mktemp -d "$work_parent/git-remote-e2ee-compare.XXXXXX") +cleanup() { + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + return + fi + case "$work" in + "$work_parent"/git-remote-e2ee-compare.*) rm -rf -- "$work" ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +tree_allocated_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + else + du -sk "$path" | awk '{print $1 * 1024}' + fi +} + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {print sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {print sum + 0}' + fi +} + +object_snapshot() { + local root=$1 + local object digest + find "$root/objects" -type f -print | LC_ALL=C sort | while IFS= read -r object; do + if [[ $time_style == bsd ]]; then + digest=$(shasum -a 256 "$object" | awk '{print $1}') + else + digest=$(sha256sum "$object" | awk '{print $1}') + fi + printf '%s\t%s\n' "${object#"$root"/}" "$digest" + done +} + +parse_elapsed_gnu() { + awk -F': ' '/Elapsed \(wall clock\)/ { + n=split($2, a, ":"); + if (n == 3) print a[1] * 3600 + a[2] * 60 + a[3]; + else print a[1] * 60 + a[2]; + }' "$1" +} + +phases_tsv="$results_dir/phases.tsv" +printf 'scenario\ttransport\toperation\twall_seconds\tpeak_rss_bytes\tremote_allocated_bytes\tremote_allocated_delta_bytes\tremote_logical_bytes\tremote_logical_delta_bytes\tclient_allocated_bytes\tclient_allocated_delta_bytes\n' >"$phases_tsv" + +run_phase() { + local scenario=$1 + local transport=$2 + local operation=$3 + local remote_path=$4 + local client_path=$5 + shift 5 + + local label="${scenario}-${transport}-${operation}" + local timing="$work/$label.time" + local stdout="$work/$label.stdout" + local stderr="$work/$label.stderr" + local remote_allocated_before remote_logical_before client_allocated_before + local wall rss + remote_allocated_before=$(tree_allocated_bytes "$remote_path") + remote_logical_before=$(tree_logical_bytes "$remote_path") + client_allocated_before=$(tree_allocated_bytes "$client_path") + + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,80p' "$stderr" >&2 + return 1 + fi + wall=$(awk '/ real / {print $1; exit}' "$timing") + rss=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,80p' "$stderr" >&2 + return 1 + fi + wall=$(parse_elapsed_gnu "$timing") + rss_kib=$(awk -F': ' '/Maximum resident set size/ {print $2; exit}' "$timing") + rss=$((rss_kib * 1024)) + fi + + local remote_allocated_after remote_logical_after client_allocated_after + remote_allocated_after=$(tree_allocated_bytes "$remote_path") + remote_logical_after=$(tree_logical_bytes "$remote_path") + client_allocated_after=$(tree_allocated_bytes "$client_path") + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$scenario" "$transport" "$operation" "$wall" "$rss" \ + "$remote_allocated_after" "$((remote_allocated_after - remote_allocated_before))" \ + "$remote_logical_after" "$((remote_logical_after - remote_logical_before))" \ + "$client_allocated_after" "$((client_allocated_after - client_allocated_before))" \ + >>"$phases_tsv" +} + +make_commit() { + local scenario=$1 + local old_head tree index_file new_head + old_head=$(git -C "$work/source" rev-parse "$source_ref") + tree=$(git -C "$work/source" rev-parse "$old_head^{tree}") + index_file="$work/${scenario}.index" + GIT_INDEX_FILE="$index_file" git -C "$work/source" read-tree "$tree" + + case $scenario in + tiny_change) + blob=$(printf 'git-remote-e2ee comparison benchmark tiny change\n' | + git -C "$work/source" hash-object -w --stdin) + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.git-e2ee-benchmark/tiny-change.txt" + ;; + add_10mib) + dd if=/dev/urandom of="$work/random-10mib.bin" \ + bs=1048576 count=10 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.git-e2ee-benchmark/random-10mib.bin" + ;; + modify_10mib_one_byte) + cp "$work/random-10mib.bin" "$work/random-10mib-mutated.bin" + old_byte=$(dd if="$work/random-10mib-mutated.bin" bs=1 skip=5242880 count=1 \ + 2>/dev/null | od -An -tu1 | tr -d ' ') + new_byte=$(((old_byte + 1) % 256)) + replacement=$(printf '%03o' "$new_byte") + printf '%b' "\\$replacement" | dd of="$work/random-10mib-mutated.bin" \ + bs=1 seek=5242880 conv=notrunc 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib-mutated.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.git-e2ee-benchmark/random-10mib.bin" + ;; + add_1000_small) + index_info="$work/${scenario}.index-info" + : >"$index_info" + for n in $(seq 1 1000); do + content=$(printf 'small benchmark file %04d\n%s\n' "$n" \ + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef') + blob=$(printf '%s' "$content" | git -C "$work/source" hash-object -w --stdin) + printf '100644 %s\t.git-e2ee-benchmark/small/file-%04d.txt\n' "$blob" "$n" \ + >>"$index_info" + done + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --index-info <"$index_info" + ;; + *) echo "unknown scenario: $scenario" >&2; return 2 ;; + esac + + tree=$(GIT_INDEX_FILE="$index_file" git -C "$work/source" write-tree) + new_head=$(printf 'benchmark: %s\n' "$scenario" | + git -C "$work/source" -c user.name=Benchmark \ + -c user.email=benchmark@example.invalid commit-tree "$tree" -p "$old_head") + git -C "$work/source" update-ref "$source_ref" "$new_head" "$old_head" +} + +echo "creating isolated source clone" >&2 +git clone --quiet --no-local --no-checkout --single-branch --branch "$source_branch" \ + "file://$source_repo" "$work/source" +git -C "$work/source" remote remove origin +git -C "$work/source" config gc.auto 0 +git -C "$work/source" config maintenance.auto false + +git init -q --bare "$work/raw.git" +git init -q "$work/raw-client" +git -C "$work/raw.git" config receive.autogc false +git -C "$work/raw.git" config maintenance.auto false +git -C "$work/raw-client" config gc.auto 0 +git -C "$work/raw-client" config maintenance.auto false +mkdir -p "$work/e2ee-storage" +git init -q "$work/e2ee-client" +git init -q "$work/e2ee-new-client" +git -C "$work/e2ee-client" config gc.auto 0 +git -C "$work/e2ee-client" config maintenance.auto false +git -C "$work/e2ee-new-client" config gc.auto 0 +git -C "$work/e2ee-new-client" config maintenance.auto false +"$git_e2ee" keygen --output "$work/repository.key.json" >/dev/null +"$git_e2ee" init --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" >/dev/null +if [[ $e2ee_push_mode == native ]]; then + git -C "$work/source" remote add private "e2ee::$work/e2ee-storage" + git -C "$work/source" config remote.private.e2ee-key "$work/repository.key.json" + e2ee_push_command=( + env "PATH=$project_root/target/release:$PATH" + git -C "$work/source" push --quiet private "$source_ref:$source_ref" + ) +else + e2ee_push_command=( + "$git_e2ee" push --storage "$work/e2ee-storage" + --key "$work/repository.key.json" --repo "$work/source" --ref "$source_ref" + ) +fi + +run_phase initial git push "$work/raw.git" "$work/source" \ + git -C "$work/source" push --quiet "$work/raw.git" "$source_ref:$source_ref" +run_phase initial e2ee push "$work/e2ee-storage" "$work/source" \ + "${e2ee_push_command[@]}" +run_phase initial git fetch "$work/raw.git" "$work/raw-client" \ + git -C "$work/raw-client" fetch --quiet "$work/raw.git" \ + "$source_ref:refs/remotes/benchmark/$source_branch" +run_phase initial e2ee fetch "$work/e2ee-storage" "$work/e2ee-client" \ + "$git_e2ee" fetch --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" --repo "$work/e2ee-client" \ + --remote-name benchmark + +repository_root=$(sed -n 's/^[[:space:]]*"repository_root": "\([^"]*\)",$/\1/p' \ + "$work/repository.key.json") +[[ -n $repository_root ]] +"$git_e2ee" keygen --repository-root "$repository_root" \ + --output "$work/new-user.key.json" >/dev/null +"$git_e2ee" device-export --key "$work/new-user.key.json" \ + --output "$work/new-user.public.json" >/dev/null +object_snapshot "$work/e2ee-storage" >"$work/objects-before-device-add.sha256" +run_phase add_user e2ee device-add "$work/e2ee-storage" "$work/e2ee-new-client" \ + "$git_e2ee" device-add --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" --device "$work/new-user.public.json" +active_reader_key="$work/new-user.key.json" +if (( total_readers > 2 )); then + for reader_number in $(seq 3 "$total_readers"); do + reader_label=$(printf '%03d' "$reader_number") + reader_key="$work/reader-$reader_label.key.json" + reader_public="$work/reader-$reader_label.public.json" + "$git_e2ee" keygen --repository-root "$repository_root" \ + --output "$reader_key" >/dev/null + "$git_e2ee" device-export --key "$reader_key" \ + --output "$reader_public" >/dev/null + run_phase "add_reader_$reader_label" e2ee device-add \ + "$work/e2ee-storage" "$work/e2ee-new-client" \ + "$git_e2ee" device-add --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" --device "$reader_public" + active_reader_key=$reader_key + done +fi +object_snapshot "$work/e2ee-storage" >"$work/objects-after-device-add.sha256" +cmp "$work/objects-before-device-add.sha256" "$work/objects-after-device-add.sha256" +run_phase add_user e2ee fetch-new-user "$work/e2ee-storage" \ + "$work/e2ee-new-client" \ + "$git_e2ee" fetch --storage "$work/e2ee-storage" \ + --key "$active_reader_key" --repo "$work/e2ee-new-client" \ + --remote-name benchmark +initial_head=$(git -C "$work/source" rev-parse "$source_ref") +new_user_head=$(git -C "$work/e2ee-new-client" \ + rev-parse "refs/remotes/benchmark/$source_branch") +[[ $new_user_head == "$initial_head" ]] + +for scenario in tiny_change add_10mib modify_10mib_one_byte add_1000_small; do + make_commit "$scenario" + run_phase "$scenario" git push "$work/raw.git" "$work/source" \ + git -C "$work/source" push --quiet "$work/raw.git" "$source_ref:$source_ref" + run_phase "$scenario" e2ee push "$work/e2ee-storage" "$work/source" \ + "${e2ee_push_command[@]}" + run_phase "$scenario" git fetch "$work/raw.git" "$work/raw-client" \ + git -C "$work/raw-client" fetch --quiet "$work/raw.git" \ + "$source_ref:refs/remotes/benchmark/$source_branch" + run_phase "$scenario" e2ee fetch "$work/e2ee-storage" "$work/e2ee-new-client" \ + "$git_e2ee" fetch --storage "$work/e2ee-storage" \ + --key "$active_reader_key" --repo "$work/e2ee-new-client" \ + --remote-name benchmark +done + +expected=$(git -C "$work/source" rev-parse "$source_ref") +raw_received=$(git -C "$work/raw-client" rev-parse "refs/remotes/benchmark/$source_branch") +e2ee_received=$(git -C "$work/e2ee-new-client" \ + rev-parse "refs/remotes/benchmark/$source_branch") +[[ $raw_received == "$expected" ]] +[[ $e2ee_received == "$expected" ]] +git -C "$work/raw-client" fsck --full --no-dangling >/dev/null +git -C "$work/e2ee-new-client" fsck --full --no-dangling >/dev/null +"$git_e2ee" verify --storage "$work/e2ee-storage" \ + --key "$work/repository.key.json" >/dev/null + +source_commits=$(git -C "$source_repo" rev-list --count HEAD) +source_files=$(git -C "$source_repo" ls-tree -r --name-only HEAD | wc -l | tr -d ' ') +source_reachable_bytes=$(git -C "$source_repo" rev-list --disk-usage --objects HEAD | + awk '{print $1}') +crate_sha=$(git -C "$project_root" rev-parse HEAD) +if [[ -n $(git -C "$project_root" status --porcelain=v1) ]]; then + crate_dirty=true +else + crate_dirty=false +fi + +cat >"$results_dir/summary.json" <&2 +cat "$phases_tsv" diff --git a/scripts/benchmark-file-filters.sh b/scripts/benchmark-file-filters.sh new file mode 100755 index 0000000..93094a4 --- /dev/null +++ b/scripts/benchmark-file-filters.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 GIT_CRYPT_CHECKOUT TRANSCRYPT_CHECKOUT [RESULTS_DIRECTORY]" >&2 +} + +if [[ $# -lt 2 || $# -gt 3 ]]; then + usage + exit 2 +fi + +git_crypt_checkout=$(cd "$1" && pwd -P) +transcrypt_checkout=$(cd "$2" && pwd -P) +git_crypt="$git_crypt_checkout/git-crypt" +transcrypt="$transcrypt_checkout/transcrypt" +[[ -x $git_crypt && -x $transcrypt ]] + +results_dir=${3:-$(pwd -P)/bench-results/file-filters-$(date -u +%Y%m%dT%H%M%SZ)} +results_dir=$(mkdir -p "$results_dir" && cd "$results_dir" && pwd -P) + +case $(uname -s) in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform: $(uname -s)" >&2; exit 2 ;; +esac + +work_parent=${BENCH_WORK_PARENT:-/tmp} +work=$(mktemp -d "$work_parent/git-remote-e2ee-filter-bench.XXXXXX") +cleanup() { + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + return + fi + case "$work" in + "$work_parent"/git-remote-e2ee-filter-bench.*) rm -rf -- "$work" ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +tree_allocated_bytes() { + local path=$1 + if [[ -e $path ]]; then du -sk "$path" | awk '{print $1 * 1024}'; else echo 0; fi +} + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {print sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {print sum + 0}' + fi +} + +parse_elapsed_gnu() { + awk -F': ' '/Elapsed \(wall clock\)/ { + n=split($2, a, ":"); + if (n == 3) print a[1] * 3600 + a[2] * 60 + a[3]; + else print a[1] * 60 + a[2]; + }' "$1" +} + +phases_tsv="$results_dir/phases.tsv" +printf 'tool\tscenario\toperation\twall_seconds\tpeak_rss_bytes\tremote_allocated_bytes\tremote_allocated_delta_bytes\tremote_logical_bytes\tremote_logical_delta_bytes\tclient_allocated_bytes\tclient_allocated_delta_bytes\n' >"$phases_tsv" + +run_phase() { + local tool=$1 scenario=$2 operation=$3 remote_path=$4 client_path=$5 + shift 5 + local label="${tool}-${scenario}-${operation}" + local timing="$work/$label.time" stdout="$work/$label.stdout" stderr="$work/$label.stderr" + local ra_before rl_before ca_before wall rss + ra_before=$(tree_allocated_bytes "$remote_path") + rl_before=$(tree_logical_bytes "$remote_path") + ca_before=$(tree_allocated_bytes "$client_path") + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(awk '/ real / {print $1; exit}' "$timing") + rss=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(parse_elapsed_gnu "$timing") + rss_kib=$(awk -F': ' '/Maximum resident set size/ {print $2; exit}' "$timing") + rss=$((rss_kib * 1024)) + fi + local ra_after rl_after ca_after + ra_after=$(tree_allocated_bytes "$remote_path") + rl_after=$(tree_logical_bytes "$remote_path") + ca_after=$(tree_allocated_bytes "$client_path") + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$tool" "$scenario" "$operation" "$wall" "$rss" "$ra_after" \ + "$((ra_after - ra_before))" "$rl_after" "$((rl_after - rl_before))" \ + "$ca_after" "$((ca_after - ca_before))" >>"$phases_tsv" +} + +make_payloads() { + mkdir -p "$work/payloads/small" + dd if=/dev/urandom of="$work/payloads/random-10mib.bin" \ + bs=1048576 count=10 2>/dev/null + cp "$work/payloads/random-10mib.bin" "$work/payloads/random-10mib-mutated.bin" + printf '\001' | dd of="$work/payloads/random-10mib-mutated.bin" \ + bs=1 seek=5242880 conv=notrunc 2>/dev/null + for n in $(seq 1 1000); do + dd if=/dev/urandom of="$work/payloads/small/file-$(printf '%04d' "$n").bin" \ + bs=1024 count=1 2>/dev/null + done +} + +commit_repo() { + local repo=$1 message=$2 + git -C "$repo" -c user.name=Benchmark -c user.email=benchmark@example.invalid \ + commit -q -m "$message" +} + +benchmark_tool() { + local tool=$1 repo="$work/$1-source" remote="$work/$1-remote.git" + local client="$work/$1-client" secret_pattern='secret/**' + local password='benchmark-only-not-a-user-secret' + + git init -q -b main "$repo" + git init -q --bare "$remote" + git -C "$remote" symbolic-ref HEAD refs/heads/main + git -C "$repo" config gc.auto 0 + git -C "$repo" config maintenance.auto false + git -C "$repo" remote add origin "$remote" + mkdir -p "$repo/secret" + + if [[ $tool == git-crypt ]]; then + (cd "$repo" && "$git_crypt" init) + printf '%s filter=git-crypt diff=git-crypt\n' "$secret_pattern" >"$repo/.gitattributes" + (cd "$repo" && "$git_crypt" export-key "$work/git-crypt.key") + else + git -C "$repo" -c advice.detachedHead=false status --short >/dev/null + (cd "$repo" && "$transcrypt" -c aes-256-cbc -p "$password" -y >/dev/null) + (cd "$repo" && "$transcrypt" --add="$secret_pattern" >/dev/null) + fi + git -C "$repo" add .gitattributes + commit_repo "$repo" 'configure encrypted file filter' + git -C "$repo" push -q -u origin main + + cp "$work/payloads/random-10mib.bin" "$repo/secret/random-10mib.bin" + run_phase "$tool" add_10mib stage "$remote" "$repo" \ + git -C "$repo" add secret/random-10mib.bin + commit_repo "$repo" 'add encrypted 10 MiB file' + run_phase "$tool" add_10mib push "$remote" "$repo" \ + git -C "$repo" push -q origin main + + cp "$work/payloads/random-10mib-mutated.bin" "$repo/secret/random-10mib.bin" + run_phase "$tool" modify_10mib_one_byte stage "$remote" "$repo" \ + git -C "$repo" add secret/random-10mib.bin + commit_repo "$repo" 'modify one byte in encrypted 10 MiB file' + run_phase "$tool" modify_10mib_one_byte push "$remote" "$repo" \ + git -C "$repo" push -q origin main + + mkdir -p "$repo/secret/small" + cp -R "$work/payloads/small/." "$repo/secret/small/" + run_phase "$tool" add_1000_small stage "$remote" "$repo" \ + git -C "$repo" add secret/small + commit_repo "$repo" 'add 1000 encrypted small files' + run_phase "$tool" add_1000_small push "$remote" "$repo" \ + git -C "$repo" push -q origin main + + plaintext_hash=$(git -C "$repo" hash-object --no-filters secret/random-10mib.bin) + ciphertext_hash=$(git -C "$repo" rev-parse HEAD:secret/random-10mib.bin) + [[ $plaintext_hash != "$ciphertext_hash" ]] + + run_phase "$tool" fresh_reader clone "$remote" "$client" \ + git clone -q "$remote" "$client" + git -C "$client" config gc.auto 0 + git -C "$client" config maintenance.auto false + if [[ $tool == git-crypt ]]; then + run_phase "$tool" fresh_reader unlock "$remote" "$client" \ + bash -c 'cd "$1" && "$2" unlock "$3"' \ + _ "$client" "$git_crypt" "$work/git-crypt.key" + else + run_phase "$tool" fresh_reader unlock "$remote" "$client" \ + bash -c 'cd "$1" && "$2" -c aes-256-cbc -p "$3" -y >/dev/null' \ + _ "$client" "$transcrypt" "$password" + fi + cmp "$work/payloads/random-10mib-mutated.bin" "$client/secret/random-10mib.bin" + git -C "$client" fsck --full --no-dangling >/dev/null +} + +make_payloads +benchmark_tool git-crypt +benchmark_tool transcrypt + +cat >"$results_dir/summary.json" <&2 +cat "$phases_tsv" diff --git a/scripts/benchmark-gcrypt.sh b/scripts/benchmark-gcrypt.sh new file mode 100755 index 0000000..ee07ed4 --- /dev/null +++ b/scripts/benchmark-gcrypt.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 SOURCE_GIT_REPOSITORY GCRYPT_CHECKOUT [RESULTS_DIRECTORY]" >&2 +} + +if [[ $# -lt 2 || $# -gt 3 ]]; then + usage + exit 2 +fi + +source_repo=$(cd "$1" && pwd -P) +gcrypt_checkout=$(cd "$2" && pwd -P) +results_dir=${3:-$(pwd -P)/bench-results/gcrypt-$(date -u +%Y%m%dT%H%M%SZ)} +results_dir=$(mkdir -p "$results_dir" && cd "$results_dir" && pwd -P) +gcrypt_helper="$gcrypt_checkout/git-remote-gcrypt" +[[ -x $gcrypt_helper ]] + +source_ref=$(git -C "$source_repo" symbolic-ref --quiet HEAD) || { + echo "source repository must have a symbolic branch checked out" >&2 + exit 2 +} +source_branch=${source_ref#refs/heads/} +source_head=$(git -C "$source_repo" rev-parse HEAD) + +case $(uname -s) in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform: $(uname -s)" >&2; exit 2 ;; +esac + +work_parent=${BENCH_WORK_PARENT:-/tmp} +work=$(mktemp -d "$work_parent/git-remote-e2ee-gcrypt-bench.XXXXXX") +cleanup() { + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + return + fi + case "$work" in + "$work_parent"/git-remote-e2ee-gcrypt-bench.*) rm -rf -- "$work" ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +tree_allocated_bytes() { + local path=$1 + if [[ -e $path ]]; then du -sk "$path" | awk '{print $1 * 1024}'; else echo 0; fi +} + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {print sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {print sum + 0}' + fi +} + +parse_elapsed_gnu() { + awk -F': ' '/Elapsed \(wall clock\)/ { + n=split($2, a, ":"); + if (n == 3) print a[1] * 3600 + a[2] * 60 + a[3]; + else print a[1] * 60 + a[2]; + }' "$1" +} + +phases_tsv="$results_dir/phases.tsv" +printf 'scenario\toperation\twall_seconds\tpeak_rss_bytes\tremote_allocated_bytes\tremote_allocated_delta_bytes\tremote_logical_bytes\tremote_logical_delta_bytes\tclient_allocated_bytes\tclient_allocated_delta_bytes\n' >"$phases_tsv" + +run_phase() { + local scenario=$1 operation=$2 remote_path=$3 client_path=$4 + shift 4 + local label="${scenario}-${operation}" + local timing="$work/$label.time" stdout="$work/$label.stdout" stderr="$work/$label.stderr" + local ra_before rl_before ca_before wall rss + ra_before=$(tree_allocated_bytes "$remote_path") + rl_before=$(tree_logical_bytes "$remote_path") + ca_before=$(tree_allocated_bytes "$client_path") + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(awk '/ real / {print $1; exit}' "$timing") + rss=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$stdout" 2>"$stderr"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$stderr" >&2 + return 1 + fi + wall=$(parse_elapsed_gnu "$timing") + rss_kib=$(awk -F': ' '/Maximum resident set size/ {print $2; exit}' "$timing") + rss=$((rss_kib * 1024)) + fi + local ra_after rl_after ca_after + ra_after=$(tree_allocated_bytes "$remote_path") + rl_after=$(tree_logical_bytes "$remote_path") + ca_after=$(tree_allocated_bytes "$client_path") + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$scenario" "$operation" "$wall" "$rss" "$ra_after" \ + "$((ra_after - ra_before))" "$rl_after" "$((rl_after - rl_before))" \ + "$ca_after" "$((ca_after - ca_before))" >>"$phases_tsv" +} + +make_commit() { + local scenario=$1 old_head tree index_file new_head blob + old_head=$(git -C "$work/source" rev-parse "$source_ref") + tree=$(git -C "$work/source" rev-parse "$old_head^{tree}") + index_file="$work/${scenario}.index" + GIT_INDEX_FILE="$index_file" git -C "$work/source" read-tree "$tree" + case $scenario in + tiny_change) + blob=$(printf 'git-remote-gcrypt comparison tiny change\n' | + git -C "$work/source" hash-object -w --stdin) + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.gcrypt-benchmark/tiny-change.txt" + ;; + add_10mib) + dd if=/dev/urandom of="$work/random-10mib.bin" \ + bs=1048576 count=10 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.gcrypt-benchmark/random-10mib.bin" + ;; + modify_10mib_one_byte) + cp "$work/random-10mib.bin" "$work/random-10mib-mutated.bin" + old_byte=$(dd if="$work/random-10mib-mutated.bin" bs=1 skip=5242880 count=1 \ + 2>/dev/null | od -An -tu1 | tr -d ' ') + new_byte=$(((old_byte + 1) % 256)) + replacement=$(printf '%03o' "$new_byte") + printf '%b' "\\$replacement" | dd of="$work/random-10mib-mutated.bin" \ + bs=1 seek=5242880 conv=notrunc 2>/dev/null + blob=$(git -C "$work/source" hash-object -w "$work/random-10mib-mutated.bin") + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --cacheinfo "100644,$blob,.gcrypt-benchmark/random-10mib.bin" + ;; + add_1000_small) + index_info="$work/${scenario}.index-info" + : >"$index_info" + for n in $(seq 1 1000); do + blob=$(printf 'small gcrypt benchmark file %04d\n' "$n" | + git -C "$work/source" hash-object -w --stdin) + printf '100644 %s\t.gcrypt-benchmark/small/file-%04d.txt\n' "$blob" "$n" \ + >>"$index_info" + done + GIT_INDEX_FILE="$index_file" git -C "$work/source" update-index \ + --add --index-info <"$index_info" + ;; + *) echo "unknown scenario: $scenario" >&2; return 2 ;; + esac + tree=$(GIT_INDEX_FILE="$index_file" git -C "$work/source" write-tree) + new_head=$(printf 'benchmark: %s\n' "$scenario" | + git -C "$work/source" -c user.name=Benchmark \ + -c user.email=benchmark@example.invalid commit-tree "$tree" -p "$old_head") + git -C "$work/source" update-ref "$source_ref" "$new_head" "$old_head" +} + +mkdir -m 700 "$work/gnupg" +export GNUPGHOME="$work/gnupg" +gpg --batch --pinentry-mode loopback --passphrase '' --quick-generate-key \ + 'Benchmark ' ed25519 sign 0 >/dev/null +fingerprint=$(gpg --batch --with-colons --list-secret-keys | + awk -F: '$1 == "fpr" {print $10; exit}') +gpg --batch --pinentry-mode loopback --passphrase '' \ + --quick-add-key "$fingerprint" cv25519 encr 0 \ + >/dev/null + +echo "creating isolated source clone" >&2 +git clone --quiet --no-local --no-checkout --single-branch --branch "$source_branch" \ + "file://$source_repo" "$work/source" +git -C "$work/source" remote remove origin +git -C "$work/source" config gc.auto 0 +git -C "$work/source" config maintenance.auto false +git -C "$work/source" remote add crypt "gcrypt::$work/remote" +git -C "$work/source" config remote.crypt.gcrypt-participants "$fingerprint" +git -C "$work/source" config remote.crypt.gcrypt-signingkey "$fingerprint" +git -C "$work/source" config remote.crypt.gcrypt-require-explicit-force-push true + +git init -q "$work/client" +git -C "$work/client" config gc.auto 0 +git -C "$work/client" config maintenance.auto false +git -C "$work/client" remote add crypt "gcrypt::$work/remote" +git -C "$work/client" config remote.crypt.gcrypt-participants "$fingerprint" +git -C "$work/client" config remote.crypt.gcrypt-signingkey "$fingerprint" + +export PATH="$gcrypt_checkout:$PATH" +run_phase initial push "$work/remote" "$work/source" \ + git -C "$work/source" push --quiet --force crypt "$source_ref:$source_ref" +run_phase initial fetch "$work/remote" "$work/client" \ + git -C "$work/client" fetch --quiet crypt \ + "$source_ref:refs/remotes/crypt/$source_branch" + +for scenario in tiny_change add_10mib modify_10mib_one_byte add_1000_small; do + make_commit "$scenario" + run_phase "$scenario" push "$work/remote" "$work/source" \ + git -C "$work/source" push --quiet --force crypt "$source_ref:$source_ref" + run_phase "$scenario" fetch "$work/remote" "$work/client" \ + git -C "$work/client" fetch --quiet crypt \ + "$source_ref:refs/remotes/crypt/$source_branch" +done + +expected=$(git -C "$work/source" rev-parse "$source_ref") +received=$(git -C "$work/client" rev-parse "refs/remotes/crypt/$source_branch") +[[ $received == "$expected" ]] +git -C "$work/client" fsck --full --no-dangling >/dev/null + +cat >"$results_dir/summary.json" <&2 +cat "$phases_tsv" diff --git a/scripts/benchmark-git-backend.sh b/scripts/benchmark-git-backend.sh new file mode 100755 index 0000000..678865e --- /dev/null +++ b/scripts/benchmark-git-backend.sh @@ -0,0 +1,756 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C + +usage() { + cat >&2 <<'EOF' +usage: benchmark-git-backend.sh GODOT_CHECKOUT GCRYPT_CHECKOUT GODOT_REVISION GCRYPT_REVISION RESULTS_DIR + +Internal measurement engine for reproduce-benchmark.sh. It requires the +explicit, revision-verified input checkouts and a results directory. +BENCH_TINY_COMMITS defaults to 5. BENCH_GCRYPT_TINY_COMMITS defaults to the +same value and may be lower for a prohibitively slow full-history run. +BENCH_ROUNDS defaults to 3. BENCH_SKIP_BUILD=1 uses existing release binaries. +BENCH_KEEP_WORK=1 retains temporary repositories and command logs under the +work directory and throwaway GPG homes under /tmp. Results contain measurement +TSV files only; no keys or command logs are written there. BENCH_TRACE=1 prints +E2EE trace lines to stderr. +BENCH_E2EE_ONLY=1 skips plain and gcrypt measurements. Set +BENCH_CARRIER_ATTR_TREE=0 to leave attr.tree unset on the local carrier +receiver, simulating servers that ignore the committed carrier attributes. +EOF +} + +if [[ $# -gt 0 && ( $1 == -h || $1 == --help ) ]]; then + usage + exit 0 +fi +if [[ $# -ne 5 ]]; then + usage + exit 2 +fi + +project_root=$(cd "$(dirname "$0")/.." && pwd -P) +source_repo=$1 +gcrypt_checkout=$2 +source_revision=$3 +gcrypt_revision=$4 +results_dir=$5 +source_repo=$(cd "$source_repo" && pwd -P) +gcrypt_checkout=$(cd "$gcrypt_checkout" && pwd -P) +results_dir=$(cd "$results_dir" && pwd -P) +[[ -d $results_dir ]] || { + echo "results directory does not exist: $results_dir" >&2 + exit 2 +} +for result_file in raw.tsv medians.tsv; do + [[ ! -e $results_dir/$result_file ]] || { + echo "results file already exists: $results_dir/$result_file" >&2 + exit 2 + } +done +git_config_count=${GIT_CONFIG_COUNT:-0} +if [[ ! $git_config_count =~ ^[0-9]+$ ]]; then + echo "GIT_CONFIG_COUNT must be a non-negative integer" >&2 + exit 2 +fi +git_config_count=$((10#$git_config_count)) +printf -v git_config_key_name 'GIT_CONFIG_KEY_%d' "$git_config_count" +printf -v git_config_value_name 'GIT_CONFIG_VALUE_%d' "$git_config_count" +printf -v "$git_config_key_name" '%s' init.defaultBranch +printf -v "$git_config_value_name" '%s' main +export "$git_config_key_name" "$git_config_value_name" +export GIT_CONFIG_COUNT=$((git_config_count + 1)) +e2ee_revision=$(git -C "$project_root" rev-parse HEAD) +tiny_commits=${BENCH_TINY_COMMITS:-5} +gcrypt_tiny_commits=${BENCH_GCRYPT_TINY_COMMITS:-$tiny_commits} +rounds=${BENCH_ROUNDS:-3} +e2ee_only=${BENCH_E2EE_ONLY:-0} +carrier_attr_tree=${BENCH_CARRIER_ATTR_TREE:-1} + +if [[ $e2ee_only != 0 && $e2ee_only != 1 ]]; then + echo "BENCH_E2EE_ONLY must be 0 or 1" >&2 + exit 2 +fi +if [[ $carrier_attr_tree != 0 && $carrier_attr_tree != 1 ]]; then + echo "BENCH_CARRIER_ATTR_TREE must be 0 or 1" >&2 + exit 2 +fi +transports=(plain gcrypt e2ee) +if [[ $e2ee_only == 1 ]]; then + transports=(e2ee) +fi + +if [[ ! $tiny_commits =~ ^[1-5]$ ]]; then + echo "BENCH_TINY_COMMITS must be from 1 to 5" >&2 + exit 2 +fi +if [[ ! $gcrypt_tiny_commits =~ ^[1-5]$ ]] || (( gcrypt_tiny_commits > tiny_commits )); then + echo "BENCH_GCRYPT_TINY_COMMITS must be from 1 to BENCH_TINY_COMMITS" >&2 + exit 2 +fi +if [[ ! $rounds =~ ^[1-3]$ ]]; then + echo "BENCH_ROUNDS must be from 1 to 3" >&2 + exit 2 +fi +[[ $(git -C "$source_repo" rev-parse HEAD) == "$source_revision" ]] || { + echo "Godot checkout must be at $source_revision" >&2 + exit 2 +} +[[ $(git -C "$source_repo" rev-parse --is-shallow-repository) == false ]] || { + echo "Godot checkout must contain full history, not a shallow clone" >&2 + exit 2 +} +[[ $(git -C "$gcrypt_checkout" rev-parse HEAD) == "$gcrypt_revision" ]] || { + echo "git-remote-gcrypt checkout must be at $gcrypt_revision" >&2 + exit 2 +} +[[ -x $gcrypt_checkout/git-remote-gcrypt ]] || { + echo "git-remote-gcrypt script is missing or not executable" >&2 + exit 1 +} +git -C "$source_repo" cat-file -e HEAD:README.md || { + echo "benchmark source must contain README.md" >&2 + exit 2 +} + +gpg_bin=$(command -v gpg || true) +[[ -n $gpg_bin && -x $gpg_bin ]] || { + echo "GnuPG executable 'gpg' was not found on PATH" >&2 + exit 1 +} +os=$(uname -s) +case $os in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported platform: $os" >&2; exit 2 ;; +esac +[[ -x /usr/bin/time ]] || { + echo "/usr/bin/time is required" >&2 + exit 1 +} +if [[ $os == Linux ]]; then + default_tmp=${TMPDIR:-/var/tmp} +else + default_tmp=${TMPDIR:-/tmp} +fi +work_parent=${BENCH_WORK_PARENT:-$default_tmp} +[[ -d $work_parent ]] || { + echo "BENCH_WORK_PARENT must already exist: $work_parent" >&2 + exit 2 +} +work_parent=$(cd "$work_parent" && pwd -P) +allowed_work_parent=false +case $os in + Darwin) + allowed_roots=(/private/tmp "$default_tmp") + allowed_work_description="/private/tmp or TMPDIR" + ;; + Linux) + allowed_roots=(/var/tmp /tmp "$default_tmp") + allowed_work_description="/var/tmp, /tmp, or TMPDIR" + ;; +esac +for allowed_root in "${allowed_roots[@]}"; do + [[ -n $allowed_root ]] || continue + [[ -d $allowed_root ]] || continue + allowed_root=$(cd "$allowed_root" && pwd -P) + if [[ $work_parent == "$allowed_root" || $work_parent == "$allowed_root"/* ]]; then + allowed_work_parent=true + break + fi +done +if [[ $allowed_work_parent != true ]]; then + echo "benchmark work parent must resolve under $allowed_work_description: $work_parent" >&2 + exit 2 +fi +case $os in + Darwin) + work_filesystem_device=$(df -P "$work_parent" | awk 'NR == 2 { print $1 }') + work_filesystem=$(mount | awk -v device="$work_filesystem_device" '$1 == device { + if (match($0, /\([^,]+/)) { + print substr($0, RSTART + 1, RLENGTH - 1) + exit + } + }') + ;; + Linux) + work_filesystem=$(stat -f -c '%T' "$work_parent") + ;; +esac +case $work_filesystem in + tmpfs|ramfs) + echo "benchmark work directory is on $work_filesystem: $work_parent" >&2 + echo "Choose disk-backed storage by setting TMPDIR, or set BENCH_WORK_PARENT under /var/tmp or TMPDIR." >&2 + exit 2 + ;; +esac +source_git_kib=$(du -sk "$source_repo/.git" | awk '{print $1}') +available_kib=$(df -Pk "$work_parent" | + awk 'NR == 2 {print $4}') +if (( available_kib < source_git_kib * 12 )); then + echo "need at least 12x the Godot .git size free under the benchmark temporary directory" >&2 + exit 1 +fi +gpg_home_root=$(cd /tmp && pwd -P) +gpg_agent_socket_path="$gpg_home_root/git-remote-e2ee-gnupg.XXXXXX/S.gpg-agent" +if (( ${#gpg_agent_socket_path} >= 90 )); then + echo "the canonical /tmp path is too long for a GnuPG agent socket: $gpg_home_root" >&2 + exit 2 +fi + +if [[ ${BENCH_SKIP_BUILD:-0} != 1 ]]; then + cargo build --release --bins --manifest-path "$project_root/Cargo.toml" +fi +git_e2ee="$project_root/target/release/git-e2ee" +[[ -x $git_e2ee && -x $project_root/target/release/git-remote-e2ee ]] || { + echo "release helper binaries are missing under target/release" >&2 + exit 1 +} + +export PATH="$gcrypt_checkout:$project_root/target/release:$PATH" +if [[ ${BENCH_TRACE:-0} == 1 ]]; then + export GIT_REMOTE_E2EE_TRACE=1 +fi +work=$(mktemp -d "$work_parent/git-remote-e2ee-bench.XXXXXX") +work=$(cd "$work" && pwd -P) +chmod 700 "$work" +gpgconf_bin=$(command -v gpgconf || true) +active_gpg_home= +cleanup_gpg_home() { + local path=$1 + case "$path" in + "$gpg_home_root"/git-remote-e2ee-gnupg.*) + if [[ $(cd "$(dirname "$path")" && pwd -P) == "$gpg_home_root" && -d $path && ! -L $path ]]; then + rm -rf "$path" + else + echo "refusing to remove unexpected GPG home: $path" >&2 + fi + ;; + *) echo "refusing to remove unexpected GPG home: $path" >&2 ;; + esac +} +cleanup() { + if [[ -n $active_gpg_home && -n $gpgconf_bin ]]; then + "$gpgconf_bin" --homedir "$active_gpg_home" --kill all >/dev/null 2>&1 || true + fi + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "benchmark work directory retained at $work" >&2 + if [[ -n $active_gpg_home ]]; then + echo "throwaway GPG home retained at $active_gpg_home" >&2 + fi + return + fi + if [[ -n $active_gpg_home ]]; then + cleanup_gpg_home "$active_gpg_home" + active_gpg_home= + fi + case "$work" in + "$work_parent"/git-remote-e2ee-bench.*) + if [[ $(cd "$(dirname "$work")" && pwd -P) == "$work_parent" && ! -L $work ]]; then + rm -rf "$work" + else + echo "refusing to remove unexpected benchmark path: $work" >&2 + fi + ;; + *) echo "refusing to remove unexpected benchmark path: $work" >&2 ;; + esac +} +trap cleanup EXIT + +raw_tsv="$work/measurements.tsv" +printf 'round\ttransport\tphase\titeration\twall_seconds\tmax_rss_bytes\tremote_or_stored_delta_bytes\tremote_or_stored_total_logical_bytes\tremote_or_stored_total_allocated_bytes\tclient_git_logical_bytes\tclient_git_allocated_bytes\tlocal_state_logical_bytes\tlocal_state_allocated_bytes\tclient_object_store_growth\n' >"$raw_tsv" + +tree_logical_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + elif [[ $time_style == bsd ]]; then + find "$path" -type f -exec stat -f '%z' {} + | awk '{sum += $1} END {printf "%.0f\n", sum + 0}' + else + find "$path" -type f -exec stat -c '%s' {} + | awk '{sum += $1} END {printf "%.0f\n", sum + 0}' + fi +} + +tree_allocated_bytes() { + local path=$1 + if [[ ! -e $path ]]; then + echo 0 + else + du -sk "$path" | awk '{printf "%.0f\n", $1 * 1024}' + fi +} + +git_object_db_bytes() { + local repo=$1 git_dir + if [[ ! -d $repo ]]; then + echo 0 + return + fi + git_dir=$(git -C "$repo" rev-parse --absolute-git-dir) + tree_logical_bytes "$git_dir/objects" +} + +git_bare_object_db_bytes() { + local repo=$1 + tree_logical_bytes "$repo/objects" +} + +carrier_cache_object_db_bytes() { + local root=$1 config repo value total=0 + [[ -d $root ]] || { echo 0; return; } + while IFS= read -r config; do + repo=${config%/config} + if [[ $(git -C "$repo" rev-parse --is-bare-repository 2>/dev/null || true) == true ]]; then + value=$(git_object_db_bytes "$repo") + total=$((total + value)) + fi + done < <(find "$root" -type f -name config -print) + echo "$total" +} + +measure() { + local label=$1 timing="$work/$1.time" output="$work/$1.stdout" error="$work/$1.stderr" + shift + if [[ $time_style == bsd ]]; then + if ! /usr/bin/time -l -o "$timing" "$@" >"$output" 2>"$error"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$error" >&2 + sed -n '1,100p' "$output" >&2 + return 1 + fi + MEASURED_WALL=$(awk '/ real / {print $1; exit}' "$timing") + MEASURED_RSS_BYTES=$(awk '/maximum resident set size/ {print $1; exit}' "$timing") + else + if ! /usr/bin/time -v -o "$timing" "$@" >"$output" 2>"$error"; then + echo "benchmark phase failed: $label" >&2 + sed -n '1,100p' "$error" >&2 + sed -n '1,100p' "$output" >&2 + return 1 + fi + MEASURED_WALL=$(awk -F': ' '/Elapsed \(wall clock\)/ { + n = split($2, a, ":") + if (n == 3) printf "%.3f\n", a[1] * 3600 + a[2] * 60 + a[3] + else printf "%.3f\n", a[1] * 60 + a[2] + }' "$timing") + MEASURED_RSS_BYTES=$(awk -F': ' '/Maximum resident set size \(kbytes\)/ { + printf "%.0f\n", $2 * 1024 + exit + }' "$timing") + fi + if [[ -z $MEASURED_RSS_BYTES ]]; then + echo "could not read max RSS from $timing" >&2 + return 1 + fi + if [[ ${BENCH_TRACE:-0} == 1 && $label == *e2ee* ]]; then + awk -v phase="$label" '/^git-remote-e2ee trace / { + printf "benchmark_trace phase=%s %s\n", phase, $0 + }' "$error" >&2 + fi +} + +record() { + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$@" >>"$raw_tsv" +} + +record_sizes() { + local round=$1 transport=$2 phase=$3 iteration=$4 wall=$5 max_rss=$6 + local remote_delta=$7 remote_path=$8 client_path=$9 state_path=${10} growth=${11} + local remote_logical remote_allocated client_logical=0 client_allocated=0 + local state_logical=0 state_allocated=0 + remote_logical=$(tree_logical_bytes "$remote_path") + remote_allocated=$(tree_allocated_bytes "$remote_path") + if [[ -n $client_path ]]; then + client_logical=$(tree_logical_bytes "$client_path") + client_allocated=$(tree_allocated_bytes "$client_path") + fi + if [[ -n $state_path ]]; then + state_logical=$(tree_logical_bytes "$state_path") + state_allocated=$(tree_allocated_bytes "$state_path") + fi + record "$round" "$transport" "$phase" "$iteration" "$wall" "$max_rss" \ + "$remote_delta" "$remote_logical" "$remote_allocated" \ + "$client_logical" "$client_allocated" "$state_logical" \ + "$state_allocated" "$growth" +} + +run_round() { + local round=$1 round_dir="$work/$1" + local branch=main bare_local plain_git gcrypt_dir gcrypt_git carrier_git + local gcrypt_url e2ee_url key_directory key_carrier fingerprint gpg_home bare_repo + local source_work plain_fresh gcrypt_fresh e2ee_fresh + local cache_push cache_returning wall rss before after secondary i transport + local remote_path state_path + + mkdir -m 700 "$round_dir" + source_work="$round_dir/source" + bare_local="$round_dir/plain-local.git" + plain_git="$round_dir/plain-git.git" + gcrypt_dir="$round_dir/gcrypt-directory" + gcrypt_git="$round_dir/gcrypt-git.git" + carrier_git="$round_dir/e2ee-carrier.git" + plain_fresh="$round_dir/plain-returning" + gcrypt_fresh="$round_dir/gcrypt-returning" + e2ee_fresh="$round_dir/e2ee-returning" + cache_push="$round_dir/cache/push" + cache_returning="$round_dir/cache/returning" + gpg_home=$(mktemp -d "$gpg_home_root/git-remote-e2ee-gnupg.XXXXXX") + active_gpg_home=$gpg_home + + export GNUPGHOME="$gpg_home" + if ! "$gpg_bin" --batch --pinentry-mode loopback --passphrase '' \ + --quick-generate-key 'Benchmark ' ed25519 sign 0 \ + >"$gpg_home/keygen.stdout" 2>"$gpg_home/keygen.stderr"; then + echo "throwaway GPG key generation failed" >&2 + sed -n '1,100p' "$gpg_home/keygen.stderr" >&2 + sed -n '1,100p' "$gpg_home/keygen.stdout" >&2 + return 1 + fi + fingerprint=$("$gpg_bin" --batch --with-colons --list-secret-keys | + awk -F: '$1 == "fpr" {print $10; exit}') + [[ -n $fingerprint ]] + if ! "$gpg_bin" --batch --pinentry-mode loopback --passphrase '' \ + --quick-add-key "$fingerprint" cv25519 encr 0 \ + >"$gpg_home/subkey.stdout" 2>"$gpg_home/subkey.stderr"; then + echo "throwaway GPG encryption subkey generation failed" >&2 + sed -n '1,100p' "$gpg_home/subkey.stderr" >&2 + sed -n '1,100p' "$gpg_home/subkey.stdout" >&2 + return 1 + fi + + git clone --quiet --no-local --no-checkout --single-branch --no-tags \ + "file://$source_repo" "$source_work" + git -C "$source_work" checkout --quiet -B "$branch" "$source_revision" + git -C "$source_work" checkout --quiet -- README.md + git -C "$source_work" config gc.auto 0 + git -C "$source_work" config maintenance.auto false + git -C "$source_work" config gpg.program "$gpg_bin" + git -C "$source_work" config user.name Benchmark + git -C "$source_work" config user.email benchmark@example.invalid + + for bare_repo in "$bare_local" "$plain_git" "$gcrypt_git" "$carrier_git"; do + git init --bare --quiet --initial-branch=main "$bare_repo" + git --git-dir="$bare_repo" config gc.auto 0 + git --git-dir="$bare_repo" config maintenance.auto false + git --git-dir="$bare_repo" config receive.autogc false + git --git-dir="$bare_repo" config receive.unpackLimit 0 + done + git --git-dir="$gcrypt_git" symbolic-ref HEAD refs/heads/gcrypt-carrier + if [[ $carrier_attr_tree == 1 ]]; then + git --git-dir="$carrier_git" config attr.tree refs/heads/git-remote-e2ee + else + git --git-dir="$carrier_git" config --unset-all attr.tree 2>/dev/null || true + fi + + key_directory="$round_dir/directory.key.json" + key_carrier="$round_dir/carrier.key.json" + "$git_e2ee" keygen --output "$key_directory" >/dev/null 2>&1 + "$git_e2ee" keygen --output "$key_carrier" >/dev/null 2>&1 + gcrypt_url="gcrypt::file://$gcrypt_git#gcrypt-carrier" + e2ee_url="e2ee::git+file://$carrier_git" + + git -C "$source_work" remote add plain-local "file://$bare_local" + git -C "$source_work" remote add gcrypt-directory "gcrypt::$gcrypt_dir" + git -C "$source_work" remote add plain-git "file://$plain_git" + git -C "$source_work" remote add gcrypt-git "$gcrypt_url" + git -C "$source_work" remote add e2ee-directory "e2ee::$round_dir/e2ee-directory" + git -C "$source_work" remote add e2ee-carrier "$e2ee_url" + + for remote_name in gcrypt-directory gcrypt-git; do + git -C "$source_work" config "remote.$remote_name.gcrypt-participants" "$fingerprint" + git -C "$source_work" config "remote.$remote_name.gcrypt-signingkey" "$fingerprint" + git -C "$source_work" config "remote.$remote_name.gcrypt-require-explicit-force-push" true + done + git -C "$source_work" config remote.e2ee-directory.e2ee-key "$key_directory" + git -C "$source_work" config remote.e2ee-carrier.e2ee-key "$key_carrier" + "$git_e2ee" init --storage "$round_dir/e2ee-directory" --key "$key_directory" >/dev/null + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_push" \ + "$git_e2ee" carrier-init --remote "$carrier_git" --key "$key_carrier" >/dev/null + + echo "round $round: initial local encryption and initial Git-backend pushes" >&2 + if [[ $e2ee_only != 1 ]]; then + measure "$round-initial-local-plain" \ + git -C "$source_work" push --quiet plain-local "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$bare_local") + record_sizes "$round" plain initial_encryption 0 "$wall" "$rss" \ + "$after" "$bare_local" "$source_work/.git" "" 0 + + measure "$round-initial-local-gcrypt" \ + git -C "$source_work" push --quiet --force gcrypt-directory "$branch:master" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(tree_logical_bytes "$gcrypt_dir") + record_sizes "$round" gcrypt initial_encryption 0 "$wall" "$rss" \ + "$after" "$gcrypt_dir" "$source_work/.git" "$source_work/.git/remote-gcrypt" 0 + fi + + measure "$round-initial-local-e2ee" \ + git -C "$source_work" push --quiet e2ee-directory "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(tree_logical_bytes "$round_dir/e2ee-directory") + record_sizes "$round" e2ee initial_encryption 0 "$wall" "$rss" \ + "$after" "$round_dir/e2ee-directory" "$source_work/.git" "" 0 + + if [[ $e2ee_only != 1 ]]; then + before=$(git_bare_object_db_bytes "$plain_git") + measure "$round-initial-plain-push" \ + git -C "$source_work" push --quiet plain-git "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$plain_git") + record_sizes "$round" plain initial_push 0 "$wall" "$rss" \ + "$((after - before))" "$plain_git" "$source_work/.git" "" 0 + + before=$(git_bare_object_db_bytes "$gcrypt_git") + measure "$round-initial-gcrypt-git-push" \ + git -C "$source_work" push --quiet --force gcrypt-git "$branch:master" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$gcrypt_git") + record_sizes "$round" gcrypt initial_push 0 "$wall" "$rss" \ + "$((after - before))" "$gcrypt_git" "$source_work/.git" \ + "$source_work/.git/remote-gcrypt" 0 + fi + + before=$(git_bare_object_db_bytes "$carrier_git") + measure "$round-initial-e2ee-carrier-push" \ + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_push" \ + git -C "$source_work" push --quiet e2ee-carrier "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$carrier_git") + record_sizes "$round" e2ee initial_push 0 "$wall" "$rss" \ + "$((after - before))" "$carrier_git" "$source_work/.git" "$cache_push" 0 + + echo "round $round: fresh no-checkout clones from Git backends" >&2 + if [[ $e2ee_only != 1 ]]; then + measure "$round-fresh-plain-clone" git \ + -c gc.auto=0 -c maintenance.auto=false \ + clone --quiet --no-local --no-tags --no-checkout --branch "$branch" \ + "file://$plain_git" "$plain_fresh" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_object_db_bytes "$plain_fresh") + record_sizes "$round" plain fresh_fetch 0 "$wall" "$rss" \ + "$(git_bare_object_db_bytes "$plain_git")" "$plain_git" \ + "$plain_fresh/.git" "" "$after" + + measure "$round-fresh-gcrypt-clone" git \ + -c gc.auto=0 -c maintenance.auto=false -c "gpg.program=$gpg_bin" \ + -c "remote.origin.gcrypt-participants=$fingerprint" \ + -c "remote.origin.gcrypt-signingkey=$fingerprint" \ + clone --quiet --no-local --no-tags --no-checkout --branch master \ + "$gcrypt_url" "$gcrypt_fresh" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + git -C "$gcrypt_fresh" config gc.auto 0 + git -C "$gcrypt_fresh" config maintenance.auto false + after=$(git_object_db_bytes "$gcrypt_fresh") + record_sizes "$round" gcrypt fresh_fetch 0 "$wall" "$rss" \ + "$(git_bare_object_db_bytes "$gcrypt_git")" "$gcrypt_git" \ + "$gcrypt_fresh/.git" "$gcrypt_fresh/.git/remote-gcrypt" "$after" + fi + + before=$(carrier_cache_object_db_bytes "$cache_returning") + measure "$round-fresh-e2ee-clone" env \ + "GIT_REMOTE_E2EE_CACHE_DIR=$cache_returning" \ + git -c gc.auto=0 -c maintenance.auto=false -c "e2ee.key=$key_carrier" \ + clone --quiet --no-local --no-tags --no-checkout --branch "$branch" \ + "$e2ee_url" "$e2ee_fresh" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + if [[ $e2ee_only != 1 ]]; then + git -C "$plain_fresh" config gc.auto 0 + git -C "$plain_fresh" config maintenance.auto false + fi + git -C "$e2ee_fresh" config gc.auto 0 + git -C "$e2ee_fresh" config maintenance.auto false + after=$(carrier_cache_object_db_bytes "$cache_returning") + secondary=$(git_object_db_bytes "$e2ee_fresh") + record_sizes "$round" e2ee fresh_fetch 0 "$wall" "$rss" \ + "$((after - before))" "$carrier_git" "$e2ee_fresh/.git" \ + "$cache_returning" "$secondary" + + for ((i = 1; i <= tiny_commits; i++)); do + local plain_added=0 gcrypt_added=0 e2ee_added=0 + measure "$round-tiny-commit-$i" bash -c ' + set -euo pipefail + repo=$1 + iteration=$2 + readme="$repo/README.md" + temporary="$repo/README.md.benchmark" + { printf "Godot Git backend benchmark update %s\\n" "$iteration"; tail -n +2 "$readme"; } >"$temporary" + mv "$temporary" "$readme" + git -C "$repo" add -- README.md + git -C "$repo" commit --quiet -m "benchmark: tiny update $iteration" + ' _ "$source_work" "$i" + for transport in "${transports[@]}"; do + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + case $transport in + plain) remote_path=$plain_git; state_path= ;; + gcrypt) remote_path=$gcrypt_git; state_path="$source_work/.git/remote-gcrypt" ;; + e2ee) remote_path=$carrier_git; state_path=$cache_push ;; + esac + record_sizes "$round" "$transport" tiny_commit "$i" "$wall" "$rss" \ + 0 "$remote_path" "$source_work/.git" "$state_path" 0 + done + + if [[ $e2ee_only != 1 ]]; then + before=$(git_bare_object_db_bytes "$plain_git") + measure "$round-tiny-plain-push-$i" \ + git -C "$source_work" push --quiet plain-git "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$plain_git") + plain_added=$((after - before)) + record_sizes "$round" plain tiny_push "$i" "$wall" "$rss" \ + "$plain_added" "$plain_git" "$source_work/.git" "" 0 + fi + + if [[ $e2ee_only != 1 ]] && (( i <= gcrypt_tiny_commits )); then + before=$(git_bare_object_db_bytes "$gcrypt_git") + measure "$round-tiny-gcrypt-push-$i" \ + git -C "$source_work" push --quiet --force gcrypt-git "$branch:master" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$gcrypt_git") + gcrypt_added=$((after - before)) + record_sizes "$round" gcrypt tiny_push "$i" "$wall" "$rss" \ + "$gcrypt_added" "$gcrypt_git" "$source_work/.git" \ + "$source_work/.git/remote-gcrypt" 0 + fi + + before=$(git_bare_object_db_bytes "$carrier_git") + measure "$round-tiny-e2ee-push-$i" \ + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_push" \ + git -C "$source_work" push --quiet e2ee-carrier "$branch:$branch" + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_bare_object_db_bytes "$carrier_git") + e2ee_added=$((after - before)) + record_sizes "$round" e2ee tiny_push "$i" "$wall" "$rss" \ + "$e2ee_added" "$carrier_git" "$source_work/.git" "$cache_push" 0 + + if [[ $e2ee_only != 1 ]]; then + before=$(git_object_db_bytes "$plain_fresh") + measure "$round-tiny-plain-fetch-$i" \ + git -C "$plain_fresh" fetch --quiet origin + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_object_db_bytes "$plain_fresh") + record_sizes "$round" plain tiny_update "$i" "$wall" "$rss" \ + "$plain_added" "$plain_git" "$plain_fresh/.git" "" \ + "$((after - before))" + fi + + if [[ $e2ee_only != 1 ]] && (( i <= gcrypt_tiny_commits )); then + before=$(git_object_db_bytes "$gcrypt_fresh") + measure "$round-tiny-gcrypt-fetch-$i" \ + git -C "$gcrypt_fresh" fetch --quiet origin + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(git_object_db_bytes "$gcrypt_fresh") + record_sizes "$round" gcrypt tiny_update "$i" "$wall" "$rss" \ + "$gcrypt_added" "$gcrypt_git" "$gcrypt_fresh/.git" \ + "$gcrypt_fresh/.git/remote-gcrypt" "$((after - before))" + fi + + before=$(carrier_cache_object_db_bytes "$cache_returning") + secondary_before=$(git_object_db_bytes "$e2ee_fresh") + measure "$round-tiny-e2ee-fetch-$i" \ + env "GIT_REMOTE_E2EE_CACHE_DIR=$cache_returning" \ + git -C "$e2ee_fresh" fetch --quiet origin + wall=$MEASURED_WALL + rss=$MEASURED_RSS_BYTES + after=$(carrier_cache_object_db_bytes "$cache_returning") + secondary=$(git_object_db_bytes "$e2ee_fresh") + record_sizes "$round" e2ee tiny_update "$i" "$wall" "$rss" \ + "$e2ee_added" "$carrier_git" "$e2ee_fresh/.git" \ + "$cache_returning" "$((secondary - secondary_before))" + done + + if [[ $e2ee_only != 1 ]]; then + [[ $(git -C "$plain_fresh" rev-parse "refs/remotes/origin/$branch") == \ + $(git -C "$source_work" rev-parse "$branch") ]] + [[ $(git -C "$gcrypt_fresh" rev-parse refs/remotes/origin/master) == \ + $(git -C "$source_work" rev-parse "$branch~$((tiny_commits - gcrypt_tiny_commits))") ]] + fi + [[ $(git -C "$e2ee_fresh" rev-parse "refs/remotes/origin/$branch") == \ + $(git -C "$source_work" rev-parse "$branch") ]] + + if [[ -n $gpgconf_bin ]]; then + "$gpgconf_bin" --homedir "$gpg_home" --kill all >/dev/null 2>&1 || true + fi + unset GNUPGHOME + if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "throwaway GPG home retained at $gpg_home" >&2 + else + cleanup_gpg_home "$gpg_home" + fi + active_gpg_home= + if [[ ${BENCH_KEEP_WORK:-0} != 1 ]]; then + case "$round_dir" in + "$work"/[1-3]) + if [[ -d $round_dir && ! -L $round_dir ]]; then + rm -rf "$round_dir" + fi + ;; + *) echo "refusing to remove unexpected round path: $round_dir" >&2 ;; + esac + fi +} + +median_value() { + local transport=$1 phase=$2 field=$3 iteration=${4:-} + awk -F '\t' -v t="$transport" -v p="$phase" -v i="$iteration" -v f="$field" ' + NR > 1 && $2 == t && $3 == p && (i == "" || $4 == i) { print $f } + ' "$raw_tsv" | sort -n | awk ' + { values[NR] = $1 } + END { + if (NR == 0) { print "-"; exit } + if (NR % 2 == 1) printf "%.3f", values[(NR + 1) / 2] + else printf "%.3f", (values[NR / 2] + values[NR / 2 + 1]) / 2 + } + ' +} + +for ((round = 1; round <= rounds; round++)); do + run_round "$round" +done + +medians_tsv="$work/medians.tsv" +printf 'transport\tphase\titeration\tmedian_wall_seconds\tmedian_max_rss_bytes\tmedian_remote_delta_bytes\tmedian_remote_total_logical_bytes\tmedian_remote_total_allocated_bytes\tmedian_client_git_logical_bytes\tmedian_client_git_allocated_bytes\tmedian_local_state_logical_bytes\tmedian_local_state_allocated_bytes\tmedian_client_object_store_growth\n' >"$medians_tsv" +for phase in initial_encryption initial_push fresh_fetch tiny_commit tiny_push tiny_update; do + for transport in "${transports[@]}"; do + printf '%s\t%s\tall' "$transport" "$phase" >>"$medians_tsv" + for field in 5 6 7 8 9 10 11 12 13 14; do + printf '\t%s' "$(median_value "$transport" "$phase" "$field")" >>"$medians_tsv" + done + printf '\n' >>"$medians_tsv" + done +done + +for phase in tiny_push tiny_update; do + for ((i = 1; i <= tiny_commits; i++)); do + for transport in "${transports[@]}"; do + printf '%s\t%s\t%s' "$transport" "$phase" "$i" >>"$medians_tsv" + for field in 5 6 7 8 9 10 11 12 13 14; do + printf '\t%s' "$(median_value "$transport" "$phase" "$field" "$i")" >>"$medians_tsv" + done + printf '\n' >>"$medians_tsv" + done + done +done + +cp "$raw_tsv" "$results_dir/raw.tsv" +cp "$medians_tsv" "$results_dir/medians.tsv" +printf 'Wrote %s and %s\n' "$results_dir/raw.tsv" "$results_dir/medians.tsv" +if [[ ${BENCH_KEEP_WORK:-0} == 1 ]]; then + echo "raw command logs, repositories, and GPG homes retained under $work" >&2 +else + echo "raw command logs, repositories, and GPG homes were removed from $work" >&2 +fi diff --git a/scripts/reproduce-benchmark.sh b/scripts/reproduce-benchmark.sh new file mode 100755 index 0000000..95b7e6c --- /dev/null +++ b/scripts/reproduce-benchmark.sh @@ -0,0 +1,579 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +export LC_ALL=C + +usage() { + cat <<'EOF' +usage: scripts/reproduce-benchmark.sh + +Fetches the pinned Godot and git-remote-gcrypt histories, runs the full +three-round benchmark, and writes raw.tsv, medians.tsv, environment.json, and +summary.md under ./bench-results/-/. + +Set BENCH_CACHE_DIR to override the input cache. Set BENCH_RESULTS_DIR to +choose an exact results directory. Set BENCH_WORK_PARENT to choose a temporary +work directory under an allowed temporary root. Linux defaults to /var/tmp; +TMPDIR overrides the default when set. tmpfs and ramfs work directories are +rejected. BENCH_ROUNDS defaults to 3, BENCH_TINY_COMMITS defaults to 5, and +BENCH_GCRYPT_TINY_COMMITS defaults to BENCH_TINY_COMMITS. +EOF +} + +if [[ $# -gt 0 ]]; then + if [[ $# == 1 && ( $1 == -h || $1 == --help ) ]]; then + usage + exit 0 + fi + usage >&2 + exit 2 +fi + +project_root=$(cd "$(dirname "$0")/.." && pwd -P) +godot_url=https://github.com/godotengine/godot.git +gcrypt_url=https://github.com/spwhitton/git-remote-gcrypt.git +godot_revision=00932449c9f372b30301d8b5fdc1be70ec12b5c0 +gcrypt_revision=a5ff704d071f14b95b6b1fa0caa8cdbf0c6cdadb + +rounds=${BENCH_ROUNDS:-3} +tiny_commits=${BENCH_TINY_COMMITS:-5} +gcrypt_tiny_commits=${BENCH_GCRYPT_TINY_COMMITS:-$tiny_commits} +e2ee_only=${BENCH_E2EE_ONLY:-0} +carrier_attr_tree=${BENCH_CARRIER_ATTR_TREE:-1} +trace=${BENCH_TRACE:-0} +skip_build=${BENCH_SKIP_BUILD:-0} +keep_work=${BENCH_KEEP_WORK:-0} + +if [[ ! $rounds =~ ^[1-3]$ ]]; then + echo "BENCH_ROUNDS must be from 1 to 3" >&2 + exit 2 +fi +if [[ ! $tiny_commits =~ ^[1-5]$ ]]; then + echo "BENCH_TINY_COMMITS must be from 1 to 5" >&2 + exit 2 +fi +if [[ ! $gcrypt_tiny_commits =~ ^[1-5]$ ]] || (( gcrypt_tiny_commits > tiny_commits )); then + echo "BENCH_GCRYPT_TINY_COMMITS must be from 1 to BENCH_TINY_COMMITS" >&2 + exit 2 +fi +validate_switch() { + local name=$1 value=$2 + if [[ $value != 0 && $value != 1 ]]; then + echo "$name must be 0 or 1" >&2 + exit 2 + fi +} +validate_switch BENCH_E2EE_ONLY "$e2ee_only" +validate_switch BENCH_CARRIER_ATTR_TREE "$carrier_attr_tree" +validate_switch BENCH_TRACE "$trace" +validate_switch BENCH_SKIP_BUILD "$skip_build" +validate_switch BENCH_KEEP_WORK "$keep_work" + +os=$(uname -s) +case $os in + Darwin) time_style=bsd ;; + Linux) time_style=gnu ;; + *) echo "unsupported operating system: $os (supported: macOS and Linux)" >&2; exit 2 ;; +esac + +missing=() +for tool in git gpg cargo rustc du df awk find sort stat mktemp hostname date sed tr tail wc chmod cp mv mkdir rmdir rm cat dirname; do + if ! command -v "$tool" >/dev/null 2>&1; then + printf "missing required command on PATH: %s\n" "$tool" >&2 + missing+=("$tool") + fi +done +if [[ $os == Darwin ]] && ! command -v mount >/dev/null 2>&1; then + echo "missing required command on PATH: mount" >&2 + missing+=(mount) +fi +if [[ ! -x /usr/bin/time ]]; then + echo "missing required command: /usr/bin/time" >&2 + missing+=(time) +fi +if (( ${#missing[@]} > 0 )); then + echo "Install the missing tools, then rerun the benchmark." >&2 + echo "Debian/Ubuntu: sudo apt-get install git gnupg cargo rustc time coreutils findutils gawk sed" >&2 + echo "Homebrew: brew install git gnupg rust gawk" >&2 + exit 1 +fi + +case $time_style in + bsd) + if ! /usr/bin/time -l -o /dev/null true >/dev/null 2>&1; then + echo "/usr/bin/time must support BSD -l output on macOS" >&2 + echo "Debian/Ubuntu: sudo apt-get install time" >&2 + echo "Homebrew: macOS normally supplies BSD /usr/bin/time; restore the system tool." >&2 + exit 1 + fi + ;; + gnu) + if ! /usr/bin/time -v -o /dev/null true >/dev/null 2>&1; then + echo "/usr/bin/time must support GNU -v output on Linux" >&2 + echo "Debian/Ubuntu: sudo apt-get install time" >&2 + echo "Homebrew: brew install gnu-time" >&2 + exit 1 + fi + ;; +esac + +gpg_bin=$(command -v gpg) +git_version=$(git --version) +gpg_version=$(gpg --version | sed -n '1p') +rustc_version=$(rustc --version) +cargo_version=$(cargo --version) +e2ee_revision=$(git -C "$project_root" rev-parse HEAD) +if [[ -n $(git -C "$project_root" status --porcelain --untracked-files=normal) ]]; then + e2ee_dirty=true +else + e2ee_dirty=false +fi + +if [[ $os == Linux ]]; then + default_tmp=${TMPDIR:-/var/tmp} +else + default_tmp=${TMPDIR:-/tmp} +fi +work_parent=${BENCH_WORK_PARENT:-$default_tmp} +[[ -d $work_parent ]] || { + echo "BENCH_WORK_PARENT must already exist: $work_parent" >&2 + exit 2 +} +work_parent=$(cd "$work_parent" && pwd -P) +allowed_work_parent=false +case $os in + Darwin) + allowed_roots=(/private/tmp "$default_tmp") + allowed_work_description="/private/tmp or TMPDIR" + ;; + Linux) + allowed_roots=(/var/tmp /tmp "$default_tmp") + allowed_work_description="/var/tmp, /tmp, or TMPDIR" + ;; +esac +for allowed_root in "${allowed_roots[@]}"; do + [[ -d $allowed_root ]] || continue + allowed_root=$(cd "$allowed_root" && pwd -P) + if [[ $work_parent == "$allowed_root" || $work_parent == "$allowed_root"/* ]]; then + allowed_work_parent=true + break + fi +done +if [[ $allowed_work_parent != true ]]; then + echo "BENCH_WORK_PARENT must resolve under $allowed_work_description: $work_parent" >&2 + exit 2 +fi + +case $os in + Darwin) + work_filesystem_device=$(df -P "$work_parent" | awk 'NR == 2 { print $1 }') + filesystem=$(mount | awk -v device="$work_filesystem_device" '$1 == device { + if (match($0, /\([^,]+/)) { + print substr($0, RSTART + 1, RLENGTH - 1) + exit + } + }') + ;; + Linux) + filesystem=$(stat -f -c '%T' "$work_parent") + ;; +esac +if [[ -z $filesystem ]]; then + echo "could not identify the filesystem for $work_parent" >&2 + exit 1 +fi +case $filesystem in + tmpfs|ramfs) + echo "benchmark work directory is on $filesystem: $work_parent" >&2 + echo "Choose disk-backed storage by setting TMPDIR, or set BENCH_WORK_PARENT under /var/tmp or TMPDIR." >&2 + exit 2 + ;; +esac + +cache_default=${XDG_CACHE_HOME:-${HOME:?HOME must be set}/.cache}/git-remote-e2ee-bench +cache_dir=${BENCH_CACHE_DIR:-$cache_default} +mkdir -p "$cache_dir" +cache_dir=$(cd "$cache_dir" && pwd -P) +godot_cache="$cache_dir/godot-$godot_revision" +gcrypt_cache="$cache_dir/git-remote-gcrypt-$gcrypt_revision" +stage_dir= +cache_lock= +cache_lock_owned=false +results_dir= +results_created=false +run_complete=false + +cleanup() { + if [[ -n $stage_dir ]]; then + case $stage_dir in + "$cache_dir"/.godot-fetch.*|"$cache_dir"/.gcrypt-fetch.*) + [[ ! -L $stage_dir ]] && rm -rf "$stage_dir" + ;; + *) echo "refusing to remove unexpected cache staging path: $stage_dir" >&2 ;; + esac + fi + if [[ $cache_lock_owned == true && -n $cache_lock && -d $cache_lock && ! -L $cache_lock ]]; then + rmdir "$cache_lock" 2>/dev/null || true + fi + if [[ $results_created == true && $run_complete != true && -d $results_dir ]]; then + rmdir "$results_dir" 2>/dev/null || true + fi +} +trap cleanup EXIT + +validate_single_branch_cache() { + local path=$1 revision=$2 name=$3 default_ref branch_count + [[ -d $path && ! -L $path ]] || { + echo "$name cache is not a directory: $path" >&2 + return 1 + } + [[ $(git -C "$path" rev-parse HEAD) == "$revision" ]] || { + echo "$name cache has the wrong revision at $path; refusing to modify it" >&2 + return 1 + } + [[ $(git -C "$path" rev-parse --is-shallow-repository) == false ]] || { + echo "$name cache is shallow at $path; choose a new BENCH_CACHE_DIR" >&2 + return 1 + } + default_ref=$(git -C "$path" symbolic-ref --quiet refs/remotes/origin/HEAD || true) + [[ $default_ref == refs/remotes/origin/* ]] || { + echo "$name cache does not identify its default branch: $path" >&2 + return 1 + } + branch_count=$(git -C "$path" for-each-ref --format='%(refname)' refs/remotes/origin | + awk '$0 != "refs/remotes/origin/HEAD" { count++ } END { print count + 0 }') + [[ $branch_count == 1 ]] || { + echo "$name cache must contain only its default branch; choose a new BENCH_CACHE_DIR" >&2 + return 1 + } + [[ -z $(git -C "$path" tag --list) ]] || { + echo "$name cache contains tags; choose a new BENCH_CACHE_DIR" >&2 + return 1 + } + git -C "$path" merge-base --is-ancestor "$revision" "$default_ref" || { + echo "$name pinned revision is not reachable from the cached default branch" >&2 + return 1 + } +} + +publish_cache() { + local stage=$1 destination=$2 name=$3 + cache_lock="$destination.publish-lock" + if ! mkdir "$cache_lock" 2>/dev/null; then + echo "$name cache publication is already in progress or has a stale lock: $cache_lock" >&2 + echo "If no benchmark is running, remove that lock directory and rerun." >&2 + return 1 + fi + cache_lock_owned=true + if [[ -e $destination || -L $destination ]]; then + validate_single_branch_cache "$destination" "$4" "$name" + rm -rf "$stage" + stage_dir= + else + mv "$stage/repo" "$destination" + rmdir "$stage" + stage_dir= + fi + rmdir "$cache_lock" + cache_lock= + cache_lock_owned=false +} + +if [[ -e $godot_cache || -L $godot_cache ]]; then + validate_single_branch_cache "$godot_cache" "$godot_revision" Godot + echo "Reusing Godot cache at $godot_cache" >&2 +else + echo "Fetching the full Godot default-branch history without tags" >&2 + stage_dir=$(mktemp -d "$cache_dir/.godot-fetch.XXXXXX") + git clone --quiet --no-tags --single-branch "$godot_url" "$stage_dir/repo" + git -C "$stage_dir/repo" checkout --quiet --detach "$godot_revision" + validate_single_branch_cache "$stage_dir/repo" "$godot_revision" Godot + publish_cache "$stage_dir" "$godot_cache" Godot "$godot_revision" + echo "Cached Godot at $godot_cache" >&2 +fi + +if [[ -e $gcrypt_cache || -L $gcrypt_cache ]]; then + validate_single_branch_cache "$gcrypt_cache" "$gcrypt_revision" git-remote-gcrypt + [[ -x $gcrypt_cache/git-remote-gcrypt ]] || { + echo "git-remote-gcrypt executable is missing in $gcrypt_cache" >&2 + exit 1 + } + echo "Reusing git-remote-gcrypt cache at $gcrypt_cache" >&2 +else + echo "Fetching the pinned git-remote-gcrypt history" >&2 + stage_dir=$(mktemp -d "$cache_dir/.gcrypt-fetch.XXXXXX") + git clone --quiet --no-tags --single-branch "$gcrypt_url" "$stage_dir/repo" + git -C "$stage_dir/repo" checkout --quiet --detach "$gcrypt_revision" + validate_single_branch_cache "$stage_dir/repo" "$gcrypt_revision" git-remote-gcrypt + [[ -x $stage_dir/repo/git-remote-gcrypt ]] || { + echo "git-remote-gcrypt executable is missing at the pinned revision" >&2 + exit 1 + } + publish_cache "$stage_dir" "$gcrypt_cache" git-remote-gcrypt "$gcrypt_revision" + echo "Cached git-remote-gcrypt at $gcrypt_cache" >&2 +fi + +[[ $(git -C "$godot_cache" rev-parse HEAD) == "$godot_revision" ]] +[[ $(git -C "$gcrypt_cache" rev-parse HEAD) == "$gcrypt_revision" ]] +[[ -x $gcrypt_cache/git-remote-gcrypt ]] + +if [[ -n ${BENCH_RESULTS_DIR:-} ]]; then + results_dir=$BENCH_RESULTS_DIR + [[ $results_dir == /* ]] || results_dir="$PWD/$results_dir" + mkdir -p "$(dirname "$results_dir")" + mkdir "$results_dir" +else + timestamp=$(date -u +%Y%m%dT%H%M%SZ) + host_short=$(hostname | awk -F. '{print $1}') + host_short=$(printf '%s' "$host_short" | tr -c '[:alnum:]_.-' '_') + results_dir="$project_root/bench-results/$timestamp-$host_short" + mkdir -p "$(dirname "$results_dir")" + mkdir "$results_dir" +fi +results_dir=$(cd "$results_dir" && pwd -P) +results_created=true + +echo "Running $rounds round(s) with $tiny_commits tiny commit(s)" >&2 +BENCH_GCRYPT_TINY_COMMITS=$gcrypt_tiny_commits \ + "$project_root/scripts/benchmark-git-backend.sh" \ + "$godot_cache" \ + "$gcrypt_cache" \ + "$godot_revision" \ + "$gcrypt_revision" \ + "$results_dir" >/dev/null + +os_name= +os_version= +os_build= +case $os in + Darwin) + os_name=$(sw_vers -productName) + os_version=$(sw_vers -productVersion) + os_build=$(sw_vers -buildVersion) + cpu_model=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true) + if [[ -z $cpu_model ]]; then + cpu_model=$(sysctl -n hw.model 2>/dev/null || uname -m) + fi + core_count=$(sysctl -n hw.ncpu) + ram_bytes=$(sysctl -n hw.memsize) + ;; + Linux) + os_name=$(awk -F= '$1 == "NAME" { gsub(/^"|"$/, "", $2); print $2; exit }' /etc/os-release 2>/dev/null || true) + os_version=$(awk -F= '$1 == "VERSION_ID" { gsub(/^"|"$/, "", $2); print $2; exit }' /etc/os-release 2>/dev/null || true) + [[ -n $os_name ]] || os_name=Linux + [[ -n $os_version ]] || os_version=$(uname -r) + cpu_model= + if command -v lscpu >/dev/null 2>&1; then + cpu_model=$(lscpu 2>/dev/null | awk -F: ' + { + label = tolower($1) + value = $2 + sub(/^[[:space:]]+/, "", label) + sub(/[[:space:]]+$/, "", label) + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + if (label == "model name" && model == "") model = value + if (label == "vendor id" && vendor == "") vendor = value + } + END { + if (model != "") print model + else if (vendor != "") print vendor + } + ' || true) + fi + if [[ -z $cpu_model ]]; then + cpu_model=$(awk -F: ' + { + label = tolower($1) + value = $2 + sub(/^[[:space:]]+/, "", label) + sub(/[[:space:]]+$/, "", label) + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + if (label == "model name" && model_name == "") model_name = value + if (label == "hardware" && hardware == "") hardware = value + if (label == "processor" && processor == "") processor = value + if (label == "cpu implementer" && implementer == "") implementer = value + if (label == "cpu part" && part == "") part = value + } + END { + if (model_name != "") { + print model_name + } else if (hardware != "") { + print hardware + } else if (implementer != "" && part != "") { + printf "CPU implementer %s, CPU part %s\n", implementer, part + } else if (processor != "") { + print processor + } else if (implementer != "") { + printf "CPU implementer %s\n", implementer + } else if (part != "") { + printf "CPU part %s\n", part + } + } + ' /proc/cpuinfo 2>/dev/null || true) + fi + [[ -n $cpu_model ]] || cpu_model=$(uname -m) + core_count=$(getconf _NPROCESSORS_ONLN 2>/dev/null || true) + if [[ -z $core_count ]]; then + core_count=$(awk '/^processor[[:space:]]*:/ { count++ } END { print count + 0 }' /proc/cpuinfo) + fi + ram_bytes=$(awk '/^MemTotal:/ { printf "%.0f", $2 * 1024; exit }' /proc/meminfo) + ;; +esac +kernel=$(uname -r) + +json_quote() { + local value=$1 + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + value=${value//$'\t'/\\t} + printf '"%s"' "$value" +} + +{ + printf '{\n' + printf ' "os_name": %s,\n' "$(json_quote "$os_name")" + printf ' "os_version": %s,\n' "$(json_quote "$os_version")" + printf ' "os_build": %s,\n' "$(json_quote "$os_build")" + printf ' "kernel": %s,\n' "$(json_quote "$kernel")" + printf ' "cpu_model": %s,\n' "$(json_quote "$cpu_model")" + printf ' "core_count": %s,\n' "$core_count" + printf ' "ram_bytes": %s,\n' "$ram_bytes" + printf ' "work_filesystem": %s,\n' "$(json_quote "$filesystem")" + printf ' "git_version": %s,\n' "$(json_quote "$git_version")" + printf ' "gpg_version": %s,\n' "$(json_quote "$gpg_version")" + printf ' "rustc_version": %s,\n' "$(json_quote "$rustc_version")" + printf ' "cargo_version": %s,\n' "$(json_quote "$cargo_version")" + printf ' "godot_revision": %s,\n' "$(json_quote "$godot_revision")" + printf ' "gcrypt_revision": %s,\n' "$(json_quote "$gcrypt_revision")" + printf ' "e2ee_revision": %s,\n' "$(json_quote "$e2ee_revision")" + printf ' "e2ee_uncommitted_changes": %s,\n' "$e2ee_dirty" + printf ' "benchmark_parameters": {\n' + printf ' "rounds": %s,\n' "$rounds" + printf ' "tiny_commits": %s,\n' "$tiny_commits" + printf ' "gcrypt_tiny_commits": %s,\n' "$gcrypt_tiny_commits" + printf ' "e2ee_only": %s,\n' "$([[ $e2ee_only == 1 ]] && echo true || echo false)" + printf ' "carrier_attr_tree": %s,\n' "$([[ $carrier_attr_tree == 1 ]] && echo true || echo false)" + printf ' "trace": %s,\n' "$([[ $trace == 1 ]] && echo true || echo false)" + printf ' "skip_build": %s,\n' "$([[ $skip_build == 1 ]] && echo true || echo false)" + printf ' "keep_work": %s\n' "$([[ $keep_work == 1 ]] && echo true || echo false)" + printf ' }\n' + printf '}\n' +} >"$results_dir/environment.json" + +medians_tsv="$results_dir/medians.tsv" +metric() { + local transport=$1 phase=$2 iteration=$3 column=$4 + awk -F '\t' -v t="$transport" -v p="$phase" -v i="$iteration" -v c="$column" \ + '$1 == t && $2 == p && $3 == i { print $c; exit }' "$medians_tsv" +} +total_client_disk_bytes() { + local transport=$1 git_bytes auxiliary_bytes + git_bytes=$(metric "$transport" fresh_fetch all 9) + auxiliary_bytes=$(metric "$transport" fresh_fetch all 11) + if [[ -z $git_bytes || $git_bytes == - ]]; then + printf '-' + return + fi + [[ -n $auxiliary_bytes && $auxiliary_bytes != - ]] || auxiliary_bytes=0 + awk -v git="$git_bytes" -v auxiliary="$auxiliary_bytes" 'BEGIN { printf "%.0f", git + auxiliary }' +} +format_initial_time() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { printf "%.1f s", n + 0 }'; fi +} +format_operation_time() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { if (n < 1) printf "%.2f s", n + 0; else printf "%.1f s", n + 0 }'; fi +} +format_seconds_two() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { printf "%.2f s", n + 0 }'; fi +} +format_gib() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { printf "%.2f GiB", (n + 0) / 1073741824 }'; fi +} +format_mib() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" ' + function grouped(value, digits, result) { + digits = sprintf("%.0f", value) + while (length(digits) > 3) { + result = "," substr(digits, length(digits) - 2) result + digits = substr(digits, 1, length(digits) - 3) + } + return digits result + } + BEGIN { printf "%s MiB", grouped((n + 0) / 1048576) } + '; fi +} +format_sent_bytes() { + if [[ -z $1 || $1 == - ]]; then printf '—'; else awk -v n="$1" 'BEGIN { if (n >= 1000000) printf "%.0f MB", n / 1000000; else printf "%.1f KB", n / 1000 }'; fi +} + +if [[ $e2ee_only == 1 ]]; then + common_pushes=$tiny_commits +else + common_pushes=$gcrypt_tiny_commits +fi +summary_file="$results_dir/summary.md" +{ + printf '# Benchmark summary\n\n' + printf 'Medians across %s fresh round(s). Tiny commit and update rows cover %s commit(s) per round.\n\n' "$rounds" "$tiny_commits" + printf '| | Plain Git | `git-remote-gcrypt` | `git-remote-e2ee` |\n' + printf '|---|---:|---:|---:|\n' + printf '| Initial encryption (to a local directory) | %s | %s | %s |\n' \ + "$(format_initial_time "$(metric plain initial_encryption all 4)")" \ + "$(format_initial_time "$(metric gcrypt initial_encryption all 4)")" \ + "$(format_initial_time "$(metric e2ee initial_encryption all 4)")" + printf '| Initial push | %s | %s | %s |\n' \ + "$(format_initial_time "$(metric plain initial_push all 4)")" \ + "$(format_initial_time "$(metric gcrypt initial_push all 4)")" \ + "$(format_initial_time "$(metric e2ee initial_push all 4)")" + printf '| Fresh fetch | %s | %s | %s |\n' \ + "$(format_initial_time "$(metric plain fresh_fetch all 4)")" \ + "$(format_initial_time "$(metric gcrypt fresh_fetch all 4)")" \ + "$(format_initial_time "$(metric e2ee fresh_fetch all 4)")" + printf '| Tiny commit | %s | %s | %s |\n' \ + "$(format_seconds_two "$(metric plain tiny_commit all 4)")" \ + "$(format_seconds_two "$(metric gcrypt tiny_commit all 4)")" \ + "$(format_seconds_two "$(metric e2ee tiny_commit all 4)")" + printf '| Tiny push | %s | %s | %s |\n' \ + "$(format_operation_time "$(metric plain tiny_push all 4)")" \ + "$(format_operation_time "$(metric gcrypt tiny_push all 4)")" \ + "$(format_operation_time "$(metric e2ee tiny_push all 4)")" + printf '| Tiny update (fetch) | %s | %s | %s |\n' \ + "$(format_operation_time "$(metric plain tiny_update all 4)")" \ + "$(format_operation_time "$(metric gcrypt tiny_update all 4)")" \ + "$(format_operation_time "$(metric e2ee tiny_update all 4)")" + printf '| Data sent per tiny push | %s | %s | %s |\n' \ + "$(format_sent_bytes "$(metric plain tiny_push all 6)")" \ + "$(format_sent_bytes "$(metric gcrypt tiny_push all 6)")" \ + "$(format_sent_bytes "$(metric e2ee tiny_push all 6)")" + printf '| Peak memory, initial push | %s | %s | %s |\n' \ + "$(format_gib "$(metric plain initial_push all 5)")" \ + "$(format_gib "$(metric gcrypt initial_push all 5)")" \ + "$(format_gib "$(metric e2ee initial_push all 5)")" + printf '| Peak memory, tiny push | %s | %s | %s |\n' \ + "$(format_mib "$(metric plain tiny_push all 5)")" \ + "$(format_mib "$(metric gcrypt tiny_push all 5)")" \ + "$(format_mib "$(metric e2ee tiny_push all 5)")" + tiny_push_label="tiny pushes" + if (( common_pushes == 1 )); then + tiny_push_label="tiny push" + fi + printf '| Remote size after %s %s | %s | %s | %s |\n' "$common_pushes" "$tiny_push_label" \ + "$(format_mib "$(metric plain tiny_push "$common_pushes" 7)")" \ + "$(format_mib "$(metric gcrypt tiny_push "$common_pushes" 7)")" \ + "$(format_mib "$(metric e2ee tiny_push "$common_pushes" 7)")" + printf '| Total client disk after fresh fetch (.git + auxiliary cache/state) | %s | %s | %s |\n' \ + "$(format_mib "$(total_client_disk_bytes plain)")" \ + "$(format_mib "$(total_client_disk_bytes gcrypt)")" \ + "$(format_mib "$(total_client_disk_bytes e2ee)")" + if (( common_pushes != tiny_commits )); then + printf '\nThe remote-size row uses %s pushes because gcrypt ran %s tiny pushes.\n' "$common_pushes" "$gcrypt_tiny_commits" + fi +} >"$summary_file" + +run_complete=true +printf 'Results: %s\n\n' "$results_dir" +cat "$summary_file" diff --git a/src/bin/git-remote-e2ee.rs b/src/bin/git-remote-e2ee.rs index c62abe8..b2e8c8d 100644 --- a/src/bin/git-remote-e2ee.rs +++ b/src/bin/git-remote-e2ee.rs @@ -55,6 +55,7 @@ fn run_protocol(repository: EncryptedRepository, remote_name: Str writeln!(output, "push")?; writeln!(output, "option")?; writeln!(output, "refspec refs/heads/*:refs/remotes/{remote_name}/*")?; + writeln!(output, "refspec refs/tags/*:refs/tags/*")?; writeln!(output)?; output.flush()?; } @@ -144,11 +145,13 @@ fn process_push_batch( let (source, destination) = spec .split_once(':') .context("push refspec must contain ':'")?; - if source.is_empty() { - writeln!(output, "error {destination} ref deletion is not supported")?; - continue; - } - let result = if dry_run { + let result = if source.is_empty() && dry_run { + repository + .validate_delete_ref_for_remote(local_repository, remote_name, destination) + .map(|_| String::new()) + } else if source.is_empty() { + repository.delete_ref_for_remote(local_repository, remote_name, destination) + } else if dry_run { repository .validate_push_update_for_remote( local_repository, diff --git a/src/crypto.rs b/src/crypto.rs index ba9c5b9..3beb615 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -1,11 +1,12 @@ use std::fs::{self, OpenOptions}; use std::io::{Read, Write}; use std::path::Path; +use std::time::Instant; +use aead_stream::{DecryptorBE32, EncryptorBE32, Nonce, StreamBE32}; use anyhow::{Context, Result, bail}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; -use chacha20poly1305::aead::stream::{DecryptorBE32, EncryptorBE32, Nonce, StreamBE32}; use chacha20poly1305::aead::{Aead, KeyInit, Payload}; use chacha20poly1305::{XChaCha20Poly1305, XNonce}; use ed25519_dalek::{Signature, Signer, SigningKey, VerifyingKey}; @@ -17,16 +18,21 @@ use hpke::{Deserializable, Kem, OpModeR, OpModeS, Serializable}; use rand::RngCore; use rand::rngs::OsRng; use rand_09::{SeedableRng, rngs::StdRng}; +use rand_chacha::ChaCha20Rng; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use subtle::ConstantTimeEq; use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; -const SYMMETRIC_MAGIC: &[u8; 8] = b"E2EES04\0"; -const PACK_STREAM_MAGIC: &[u8; 8] = b"E2EEPK4\0"; -const HPKE_INFO: &[u8] = b"git-remote-e2ee generation key v4"; -const KEY_COMMITMENT_DOMAIN: &[u8] = b"git-remote-e2ee generation key commitment v4\0"; -const SUBKEY_SALT: &[u8] = b"git-remote-e2ee subkey derivation v4\0"; +use crate::trace; + +const SYMMETRIC_MAGIC: &[u8; 8] = b"E2EES05\0"; +const PACK_STREAM_MAGIC: &[u8; 8] = b"E2EEPK5\0"; +const HPKE_INFO: &[u8] = b"git-remote-e2ee generation key v5"; +const KEY_COMMITMENT_DOMAIN: &[u8] = b"git-remote-e2ee generation key commitment v5\0"; +const SUBKEY_SALT: &[u8] = b"git-remote-e2ee subkey derivation v5\0"; +pub const HPKE_ENCAPSULATED_KEY_SIZE: usize = 32; +pub const HPKE_CIPHERTEXT_SIZE: usize = 32 + 16; pub const PACK_STREAM_CHUNK_SIZE: usize = 1024 * 1024; const PACK_STREAM_NONCE_SIZE: usize = 19; const PACK_STREAM_TAG_SIZE: usize = 16; @@ -44,6 +50,7 @@ pub enum SubkeyKind { ManifestBody = 1, Pack = 2, PredecessorLink = 3, + SealedHeader = 4, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -127,13 +134,9 @@ impl KeyFile { let signing_public = self.signing_key()?.verifying_key().to_bytes(); let wrapping_private = self.wrapping_private_key()?; let wrapping_public = HpkeKem::sk_to_pk(&wrapping_private).to_bytes(); - let mut identity = Vec::with_capacity(96); - identity.extend_from_slice(b"git-remote-e2ee device id v3\0"); - identity.extend_from_slice(&signing_public); - identity.extend_from_slice(&wrapping_public); Ok(PublicDevice { repository_root: self.repository_root.clone(), - device_id: hex::encode(Sha256::digest(&identity)), + device_id: device_id_for_public_keys(&signing_public, wrapping_public.as_slice()), signing_public_key: BASE64.encode(signing_public), wrapping_public_key: BASE64.encode(wrapping_public), }) @@ -240,11 +243,26 @@ pub fn wrap_generation_key( public_key: &str, generation_key: &[u8; 32], aad: &[u8], +) -> Result<(Vec, Vec)> { + wrap_generation_key_with_seed(public_key, generation_key, aad, &random_seed()) +} + +pub fn random_seed() -> [u8; 32] { + let mut seed = [0_u8; 32]; + OsRng.fill_bytes(&mut seed); + seed +} + +pub fn wrap_generation_key_with_seed( + public_key: &str, + generation_key: &[u8; 32], + aad: &[u8], + seed: &[u8; 32], ) -> Result<(Vec, Vec)> { let public_bytes = BASE64.decode(public_key)?; let public = ::PublicKey::from_bytes(&public_bytes) .map_err(|_| anyhow::anyhow!("invalid HPKE public key"))?; - let mut rng = StdRng::from_os_rng(); + let mut rng = ChaCha20Rng::from_seed(*seed); let (encapsulated, ciphertext) = hpke::single_shot_seal::( &OpModeS::Base, &public, @@ -257,6 +275,25 @@ pub fn wrap_generation_key( Ok((encapsulated.to_bytes().to_vec(), ciphertext)) } +/// Builds a padding envelope that is a genuine HPKE seal of a random key to a +/// throwaway recipient, so its encapsulated key is a real X25519 public key and +/// storage cannot tell it apart from an envelope addressed to a device. +pub fn dummy_generation_envelope() -> Result<(Vec, Vec)> { + let mut rng = ChaCha20Rng::from_seed(random_seed()); + let (_, public) = HpkeKem::gen_keypair(&mut rng); + let key = random_key(); + let (encapsulated, ciphertext) = hpke::single_shot_seal::( + &OpModeS::Base, + &public, + HPKE_INFO, + key.as_slice(), + &random_seed(), + &mut rng, + ) + .map_err(|_| anyhow::anyhow!("HPKE dummy envelope sealing failed"))?; + Ok((encapsulated.to_bytes().to_vec(), ciphertext)) +} + pub fn random_key() -> SecretKey { let mut key = [0_u8; 32]; OsRng.fill_bytes(&mut key); @@ -306,9 +343,13 @@ pub fn seal_with_key(key: &[u8; 32], plaintext: &[u8], associated_data: &[u8]) - let cipher = XChaCha20Poly1305::new(key.into()); let mut nonce = [0_u8; 24]; OsRng.fill_bytes(&mut nonce); + let nonce: &XNonce = nonce + .as_slice() + .try_into() + .expect("XChaCha20-Poly1305 nonce has the fixed expected size"); let ciphertext = cipher .encrypt( - XNonce::from_slice(&nonce), + nonce, Payload { msg: plaintext, aad: associated_data, @@ -317,7 +358,7 @@ pub fn seal_with_key(key: &[u8; 32], plaintext: &[u8], associated_data: &[u8]) - .map_err(|_| anyhow::anyhow!("encryption failed"))?; let mut envelope = Vec::with_capacity(SYMMETRIC_MAGIC.len() + nonce.len() + ciphertext.len()); envelope.extend_from_slice(SYMMETRIC_MAGIC); - envelope.extend_from_slice(&nonce); + envelope.extend_from_slice(nonce); envelope.extend_from_slice(&ciphertext); Ok(envelope) } @@ -331,9 +372,12 @@ pub fn open_with_key(key: &[u8; 32], envelope: &[u8], associated_data: &[u8]) -> let nonce_start = SYMMETRIC_MAGIC.len(); let nonce_end = nonce_start + 24; let cipher = XChaCha20Poly1305::new(key.into()); + let nonce: &XNonce = envelope[nonce_start..nonce_end] + .try_into() + .expect("validated encrypted object has the fixed nonce size"); cipher .decrypt( - XNonce::from_slice(&envelope[nonce_start..nonce_end]), + nonce, Payload { msg: &envelope[nonce_end..], aad: associated_data, @@ -353,6 +397,8 @@ struct DigestWriter { inner: W, hasher: Sha256, count: u64, + stage_write: trace::Io, + sha256: trace::Io, } impl DigestWriter { @@ -361,6 +407,8 @@ impl DigestWriter { inner, hasher: Sha256::new(), count: 0, + stage_write: trace::Io::new("pack_ciphertext_stage_write"), + sha256: trace::Io::new("pack_ciphertext_sha256"), } } @@ -371,8 +419,17 @@ impl DigestWriter { impl Write for DigestWriter { fn write(&mut self, data: &[u8]) -> std::io::Result { + let write_started = self.stage_write.is_active().then(Instant::now); let written = self.inner.write(data)?; + if let Some(started) = write_started { + self.stage_write + .record(written, started.elapsed().as_nanos()); + } + let hash_started = self.sha256.is_active().then(Instant::now); self.hasher.update(&data[..written]); + if let Some(started) = hash_started { + self.sha256.record(written, started.elapsed().as_nanos()); + } self.count += written as u64; Ok(written) } @@ -383,7 +440,7 @@ impl Write for DigestWriter { } fn pack_stream_aad(base_aad: &[u8], header: &[u8]) -> Vec { - let mut aad = b"git-remote-e2ee pack stream v4\0".to_vec(); + let mut aad = b"git-remote-e2ee pack stream v5\0".to_vec(); aad.extend_from_slice(&(base_aad.len() as u32).to_le_bytes()); aad.extend_from_slice(base_aad); aad.extend_from_slice(&(header.len() as u32).to_le_bytes()); @@ -391,11 +448,16 @@ fn pack_stream_aad(base_aad: &[u8], header: &[u8]) -> Vec { aad } -fn read_chunk(reader: &mut impl Read) -> Result> { +fn read_chunk(reader: &mut impl Read, input: &mut trace::Io) -> Result> { let mut chunk = vec![0_u8; PACK_STREAM_CHUNK_SIZE]; let mut filled = 0; while filled < chunk.len() { - match reader.read(&mut chunk[filled..]) { + let read_started = input.is_active().then(Instant::now); + let read_result = reader.read(&mut chunk[filled..]); + if let (Some(started), Ok(read)) = (read_started, &read_result) { + input.record(*read, started.elapsed().as_nanos()); + } + match read_result { Ok(0) => break, Ok(read) => filled += read, Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} @@ -412,6 +474,7 @@ pub fn seal_pack_stream( ciphertext: impl Write, base_aad: &[u8], ) -> Result { + trace::report_chacha20_backend(); let mut nonce = [0_u8; PACK_STREAM_NONCE_SIZE]; OsRng.fill_bytes(&mut nonce); let mut header = Vec::with_capacity(PACK_STREAM_HEADER_SIZE); @@ -420,19 +483,22 @@ pub fn seal_pack_stream( header.extend_from_slice(&nonce); let aad = pack_stream_aad(base_aad, &header); let stream_nonce = - Nonce::>::from_slice(&nonce); - let mut encryptor = EncryptorBE32::::new(key.into(), stream_nonce); + Nonce::>::try_from(nonce.as_slice()) + .expect("pack stream nonce has the fixed expected size"); + let mut encryptor = EncryptorBE32::::new(key.into(), &stream_nonce); let mut output = DigestWriter::new(ciphertext); + let mut input = trace::Io::new("pack_plaintext_pipe_read"); + let mut encryption = trace::Io::new("pack_stream_chacha20poly1305"); output.write_all(&header)?; - let mut current = read_chunk(&mut plaintext)?; + let mut current = read_chunk(&mut plaintext, &mut input)?; if current.is_empty() { bail!("cannot encrypt an empty pack stream") } let mut plaintext_size = 0_u64; let mut segments = 0_u64; loop { - let next = read_chunk(&mut plaintext)?; + let next = read_chunk(&mut plaintext, &mut input)?; plaintext_size = plaintext_size .checked_add(current.len() as u64) .context("pack plaintext size overflow")?; @@ -441,15 +507,23 @@ pub fn seal_pack_stream( bail!("pack stream exceeds segment counter limit") } if next.is_empty() { + let encrypt_started = encryption.is_active().then(Instant::now); encryptor .encrypt_last_in_place(&aad, &mut current) .map_err(|_| anyhow::anyhow!("pack stream encryption failed"))?; + if let Some(started) = encrypt_started { + encryption.record(current.len(), started.elapsed().as_nanos()); + } output.write_all(¤t)?; break; } + let encrypt_started = encryption.is_active().then(Instant::now); encryptor .encrypt_next_in_place(&aad, &mut current) .map_err(|_| anyhow::anyhow!("pack stream encryption failed"))?; + if let Some(started) = encrypt_started { + encryption.record(current.len(), started.elapsed().as_nanos()); + } output.write_all(¤t)?; current = next; } @@ -470,6 +544,7 @@ pub fn open_pack_stream( plaintext_size: u64, expected_id: &str, ) -> Result { + trace::report_chacha20_backend(); if plaintext_size == 0 { bail!("pack plaintext size must be nonzero") } @@ -505,12 +580,12 @@ pub fn open_pack_stream( hasher.update(header); let aad = pack_stream_aad(base_aad, &header); let nonce_start = chunk_size_start + 4; - let stream_nonce = Nonce::>::from_slice( - &header[nonce_start..], - ); + let stream_nonce = + Nonce::>::try_from(&header[nonce_start..]) + .expect("validated pack stream header has the fixed nonce size"); let mut decryptor = Some(DecryptorBE32::::new( key.into(), - stream_nonce, + &stream_nonce, )); let mut remaining = plaintext_size; let mut ciphertext_count = PACK_STREAM_HEADER_SIZE as u64; @@ -577,11 +652,7 @@ pub fn validate_public_device(device: &PublicDevice) -> Result<()> { validate_root(&device.repository_root)?; let signing = decode_public_key(&device.signing_public_key)?; let wrapping = decode_public_key(&device.wrapping_public_key)?; - let mut identity = Vec::with_capacity(96); - identity.extend_from_slice(b"git-remote-e2ee device id v3\0"); - identity.extend_from_slice(&signing); - identity.extend_from_slice(&wrapping); - if device.device_id != hex::encode(Sha256::digest(&identity)) { + if device.device_id != device_id_for_public_keys(&signing, &wrapping) { bail!("device id does not match its public keys") } Ok(()) @@ -602,6 +673,14 @@ fn repository_root_for_public_keys(signing: &[u8], wrapping: &[u8]) -> String { hex::encode(Sha256::digest(identity)) } +fn device_id_for_public_keys(signing: &[u8], wrapping: &[u8]) -> String { + let mut identity = Vec::with_capacity(32 + signing.len() + wrapping.len()); + identity.extend_from_slice(b"git-remote-e2ee device id v3\0"); + identity.extend_from_slice(signing); + identity.extend_from_slice(wrapping); + hex::encode(Sha256::digest(identity)) +} + fn validate_root(root: &str) -> Result<()> { if root.len() != 64 || !root.bytes().all(|byte| byte.is_ascii_hexdigit()) { bail!("repository root must be a 32-byte lowercase hexadecimal value") @@ -611,6 +690,19 @@ fn validate_root(root: &str) -> Result<()> { #[cfg(test)] mod tests { + + #[test] + fn dummy_envelopes_look_like_real_x25519_encapsulations() { + // A real X25519 public key never sets the top bit of its last byte. + // Uniformly random padding would set it about half the time, letting + // storage count real readers. + for _ in 0..256 { + let (encapsulated, ciphertext) = dummy_generation_envelope().unwrap(); + assert_eq!(encapsulated.len(), HPKE_ENCAPSULATED_KEY_SIZE); + assert_eq!(ciphertext.len(), HPKE_CIPHERTEXT_SIZE); + assert_eq!(encapsulated[31] & 0x80, 0); + } + } use super::*; fn pack_bytes(size: usize) -> Vec { @@ -709,6 +801,42 @@ mod tests { } } + #[test] + fn pack_stream_be32_wire_format_matches_previous_release() { + let key = [0x42_u8; 32]; + let nonce = [0x24_u8; PACK_STREAM_NONCE_SIZE]; + let aad = b"git-remote-e2ee stream compatibility"; + let mut first = b"first old-format STREAM segment".to_vec(); + let mut last = b"last old-format STREAM segment".to_vec(); + let stream_nonce = + Nonce::>::try_from(nonce.as_slice()) + .unwrap(); + let mut encryptor = EncryptorBE32::::new((&key).into(), &stream_nonce); + encryptor.encrypt_next_in_place(aad, &mut first).unwrap(); + encryptor.encrypt_last_in_place(aad, &mut last).unwrap(); + + let expected = "4bcfaa062cd22c76da334339c5ba90a0fd2a8f369e18458da2cde4848d0a662e656fddd69b2937ea72d3da289c7fb79c1afdeeff4cc3e234f31ac5454a7592e96c4dc99aa1a5473eb4dfe4dd821365b48c4738512eaf6264ee9e373823"; + assert_eq!( + format!("{}{}", hex::encode(&first), hex::encode(&last)), + expected + ); + + let ciphertext = hex::decode(expected).unwrap(); + let first_ciphertext_size = b"first old-format STREAM segment".len() + PACK_STREAM_TAG_SIZE; + let (first_ciphertext, last_ciphertext) = ciphertext.split_at(first_ciphertext_size); + let mut first_ciphertext = first_ciphertext.to_vec(); + let mut last_ciphertext = last_ciphertext.to_vec(); + let mut decryptor = DecryptorBE32::::new((&key).into(), &stream_nonce); + decryptor + .decrypt_next_in_place(aad, &mut first_ciphertext) + .unwrap(); + decryptor + .decrypt_last_in_place(aad, &mut last_ciphertext) + .unwrap(); + assert_eq!(first_ciphertext, b"first old-format STREAM segment"); + assert_eq!(last_ciphertext, b"last old-format STREAM segment"); + } + #[test] fn pack_stream_is_randomized_and_context_bound() { let plaintext = pack_bytes(PACK_STREAM_CHUNK_SIZE + 1); @@ -816,7 +944,7 @@ mod tests { ); let mut legacy = ciphertext; - legacy[..PACK_STREAM_MAGIC.len()].copy_from_slice(b"E2EEPK3\0"); + legacy[..PACK_STREAM_MAGIC.len()].copy_from_slice(b"E2EEPK4\0"); assert!(open_test_pack(&key, &legacy, sealed.plaintext_size, &sealed.object_id).is_err()); } diff --git a/src/git.rs b/src/git.rs index 1204401..0219910 100644 --- a/src/git.rs +++ b/src/git.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::fs::File; use std::io::{Read, Write}; use std::path::Path; @@ -6,6 +6,8 @@ use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; use anyhow::{Context, Result, bail}; +use crate::trace; + pub fn ensure_repository(repo: &Path) -> Result<()> { let output = Command::new("git") .arg("-C") @@ -18,10 +20,77 @@ pub fn ensure_repository(repo: &Path) -> Result<()> { Ok(()) } +pub fn has_complete_object_graph(repo: &Path) -> bool { + let shallow = match Command::new("git") + .arg("-C") + .arg(repo) + .args(["rev-parse", "--is-shallow-repository"]) + .output() + { + Ok(output) if output.status.success() => output, + _ => return false, + }; + if String::from_utf8_lossy(&shallow.stdout).trim() != "false" { + return false; + } + + let partial = match Command::new("git") + .arg("-C") + .arg(repo) + .args([ + "config", + "--local", + "--name-only", + "--get-regexp", + r"^(extensions\.partialclone|remote\..*\.promisor)$", + ]) + .output() + { + Ok(output) => output, + Err(_) => return false, + }; + partial.status.code() == Some(1) && partial.stdout.is_empty() +} + pub fn resolve_ref(repo: &Path, reference: &str) -> Result { git_text(repo, &["rev-parse", "--verify", reference]) } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InnerRefKind { + Branch, + Tag, +} + +pub fn validate_inner_ref(repo: &Path, reference: &str) -> Result { + let kind = if reference.starts_with("refs/heads/") { + InnerRefKind::Branch + } else if reference.starts_with("refs/tags/") { + InnerRefKind::Tag + } else { + bail!( + "unsupported ref namespace for {reference}; only refs/heads/* and refs/tags/* are supported" + ) + }; + let output = Command::new("git") + .arg("-C") + .arg(repo) + .args(["check-ref-format", reference]) + .output()?; + if !output.status.success() { + bail!("invalid Git ref name {reference}") + } + Ok(kind) +} + +pub fn ensure_branch_target(repo: &Path, reference: &str, object: &str) -> Result<()> { + let target = resolve_tag_chain(repo, reference, object, "pushed")?; + if target.kind != "commit" { + bail!("branch ref {reference} must resolve to a commit") + } + Ok(()) +} + pub fn is_ancestor(repo: &Path, ancestor: &str, descendant: &str) -> Result { let status = Command::new("git") .arg("-C") @@ -53,6 +122,7 @@ pub struct PackSource { stdout: Option, stderr: File, finished: bool, + _timer: trace::Span, } impl Read for PackSource { @@ -93,6 +163,7 @@ pub fn start_incremental_pack( new_refs: &BTreeMap, old_refs: &BTreeMap, ) -> Result { + let timer = trace::Span::new("git_pack_objects_stream_lifetime"); let stderr = tempfile::tempfile().context("create pack-objects stderr file")?; let mut child = Command::new("git") .arg("-C") @@ -124,6 +195,7 @@ pub fn start_incremental_pack( stdout: Some(stdout), stderr, finished: false, + _timer: timer, }) } @@ -132,6 +204,7 @@ pub struct PackImporter { stdin: Option, stderr: File, finished: bool, + _timer: trace::Span, } impl Write for PackImporter { @@ -176,6 +249,7 @@ impl Drop for PackImporter { } pub fn start_pack_import(repo: &Path) -> Result { + let timer = trace::Span::new("git_index_pack"); let stderr = tempfile::tempfile().context("create index-pack stderr file")?; let mut child = Command::new("git") .arg("-C") @@ -191,6 +265,7 @@ pub fn start_pack_import(repo: &Path) -> Result { stdin: Some(stdin), stderr, finished: false, + _timer: timer, }) } @@ -203,30 +278,65 @@ fn read_process_error(file: &mut File) -> String { message.trim().to_owned() } -pub fn ensure_refs_connected(repo: &Path, refs: &BTreeMap) -> Result<()> { +pub fn ensure_refs_connected_since( + repo: &Path, + refs: &BTreeMap, + verified_refs: &BTreeMap, +) -> Result<()> { + let _timer = trace::Span::new("git_ref_connectivity_walk"); if refs.is_empty() { return Ok(()); } + let mut commit_roots = Vec::new(); + let mut verified_commit_roots = Vec::new(); for (reference, object) in refs { - let expression = format!("{object}^{{commit}}"); - let output = Command::new("git") - .arg("-C") - .arg(repo) - .args(["cat-file", "-e", &expression]) - .output()?; - if !output.status.success() { - bail!("fetched ref {reference} does not resolve to a complete commit") + let ref_kind = validate_inner_ref(repo, reference)?; + let target = resolve_tag_chain(repo, reference, object, "fetched")?; + match target.kind.as_str() { + "commit" => commit_roots.push(target.oid), + "tree" if ref_kind == InnerRefKind::Tag => { + ensure_tree_connected(repo, reference, &target.oid)? + } + "blob" if ref_kind == InnerRefKind::Tag => {} + _ => bail!("fetched ref {reference} does not resolve to a supported Git object"), + } + } + + // A successful prior connectivity check makes every object reachable from + // these tips a trusted frontier. Git objects are immutable and + // content-addressed, so a later fetch only needs to walk objects newly + // reachable beyond that frontier. Still require the frontier tips to be + // present locally so corruption or an unexpected local prune fails closed. + for (reference, object) in verified_refs { + let ref_kind = validate_inner_ref(repo, reference)?; + let target = resolve_tag_chain(repo, reference, object, "verified")?; + match target.kind.as_str() { + "commit" => verified_commit_roots.push(target.oid), + "tree" if ref_kind == InnerRefKind::Tag => { + ensure_tree_connected(repo, reference, &target.oid)? + } + "blob" if ref_kind == InnerRefKind::Tag => {} + _ => bail!("verified ref {reference} does not resolve to a supported Git object"), } } + if commit_roots.is_empty() { + return Ok(()); + } let mut command = Command::new("git"); command .arg("-C") .arg(repo) .args(["rev-list", "--objects", "--missing=print"]); - for object in refs.values() { + for object in &commit_roots { command.arg(object); } + if !verified_commit_roots.is_empty() { + command.arg("--not"); + for object in &verified_commit_roots { + command.arg(object); + } + } let output = command.output()?; if !output.status.success() { bail!( @@ -234,11 +344,108 @@ pub fn ensure_refs_connected(repo: &Path, refs: &BTreeMap) -> Re String::from_utf8_lossy(&output.stderr).trim() ) } - if String::from_utf8(output.stdout)? - .lines() - .any(|line| line.starts_with('?')) + let output = String::from_utf8(output.stdout)?; + let mut checked_objects = 0; + for line in output.lines() { + checked_objects += 1; + if line.starts_with('?') { + bail!("fetched refs contain missing Git objects") + } + } + trace::count("git_ref_connectivity_walk_objects", checked_objects); + Ok(()) +} + +struct RefTarget { + oid: String, + kind: String, +} + +fn resolve_tag_chain(repo: &Path, reference: &str, object: &str, label: &str) -> Result { + let mut current = object.to_owned(); + let mut seen = HashSet::new(); + loop { + if !seen.insert(current.clone()) { + bail!("{label} ref {reference} contains a cyclic tag chain") + } + let kind = match cat_file_type(repo, ¤t) { + Ok(kind) => kind, + Err(_) if label == "verified" => { + bail!("verified ref {reference} is missing locally") + } + Err(_) => { + bail!("fetched ref {reference} does not resolve to a complete Git object graph") + } + }; + if kind != "tag" { + return Ok(RefTarget { oid: current, kind }); + } + let body = git_text(repo, &["cat-file", "-p", ¤t])?; + let mut target = None; + let mut target_kind = None; + for line in body.lines().take_while(|line| !line.is_empty()) { + if let Some(value) = line.strip_prefix("object ") { + if target.is_some() { + bail!("annotated tag {reference} has duplicate target objects") + } + target = Some(value); + } else if let Some(value) = line.strip_prefix("type ") { + if target_kind.is_some() { + bail!("annotated tag {reference} has duplicate target types") + } + target_kind = Some(value); + } + } + let target = target.context("annotated tag is missing its target object")?; + let expected_kind = target_kind.context("annotated tag is missing its target type")?; + if cat_file_type(repo, target)? != expected_kind { + bail!("annotated tag {reference} has a mismatched target type") + } + current = target.to_owned(); + } +} + +fn cat_file_type(repo: &Path, object: &str) -> Result { + git_text(repo, &["cat-file", "-t", object]) +} + +fn ensure_tree_connected(repo: &Path, reference: &str, tree: &str) -> Result<()> { + let output = Command::new("git") + .arg("-C") + .arg(repo) + .args(["ls-tree", "-r", "-t", "-z", "--full-tree", tree]) + .output()?; + if !output.status.success() { + bail!("fetched tag {reference} contains an incomplete tree") + } + for entry in output + .stdout + .split(|byte| *byte == 0) + .filter(|entry| !entry.is_empty()) { - bail!("fetched refs contain missing Git objects") + let header = entry + .split(|byte| *byte == b'\t') + .next() + .context("Git ls-tree returned a malformed entry")?; + let header = std::str::from_utf8(header)?; + let mut fields = header.split_ascii_whitespace(); + let mode = fields + .next() + .context("Git ls-tree returned an entry without a mode")?; + let _kind = fields + .next() + .context("Git ls-tree returned an entry without an object type")?; + let object = fields + .next() + .context("Git ls-tree returned an entry without an object ID")?; + if mode == "160000" { + // A gitlink names a submodule commit that belongs to another + // object database and is not part of this tree's connectivity. + continue; + } + if cat_file_type(repo, object).is_err() { + bail!("fetched tag {reference} contains missing Git object {object}") + } } Ok(()) } @@ -273,3 +480,68 @@ fn git_text(repo: &Path, args: &[&str]) -> Result { } Ok(String::from_utf8(output.stdout)?.trim().to_owned()) } + +#[cfg(test)] +mod tests { + use super::has_complete_object_graph; + use std::fs; + use std::path::Path; + use std::process::Command; + + fn git(repo: &Path, args: &[&str]) { + let output = Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?}: {}", + args, + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn complete_object_graph_rejects_shallow_and_promisor_repositories() { + let temporary = tempfile::tempdir().unwrap(); + let full = temporary.path().join("full"); + fs::create_dir_all(&full).unwrap(); + let init = Command::new("git") + .args(["init", "--quiet", "--initial-branch=main"]) + .arg(&full) + .output() + .unwrap(); + assert!(init.status.success()); + git(&full, &["config", "user.name", "Test"]); + git(&full, &["config", "user.email", "test@example.invalid"]); + for contents in ["first", "second"] { + fs::write(full.join("file"), contents).unwrap(); + git(&full, &["add", "file"]); + git(&full, &["commit", "--quiet", "-m", contents]); + } + assert!(has_complete_object_graph(&full)); + + let shallow = temporary.path().join("shallow"); + let source_url = format!("file://{}", full.display()); + let clone = Command::new("git") + .args(["clone", "--quiet", "--depth=1", "--no-tags"]) + .arg(source_url) + .arg(&shallow) + .output() + .unwrap(); + assert!( + clone.status.success(), + "git clone: {}", + String::from_utf8_lossy(&clone.stderr) + ); + assert!(!has_complete_object_graph(&shallow)); + + git(&full, &["config", "remote.origin.promisor", "true"]); + assert!(!has_complete_object_graph(&full)); + git(&full, &["config", "--unset", "remote.origin.promisor"]); + git(&full, &["config", "extensions.partialClone", "origin"]); + assert!(!has_complete_object_graph(&full)); + } +} diff --git a/src/lib.rs b/src/lib.rs index 9139380..456cdf6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,3 +5,4 @@ mod persist; pub mod policy; pub mod repository; pub mod storage; +mod trace; diff --git a/src/main.rs b/src/main.rs index 09ed287..13106b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,11 @@ use std::path::PathBuf; -use anyhow::Result; -use clap::{Parser, Subcommand}; +use anyhow::{Result, bail}; +use clap::{Parser, Subcommand, ValueEnum}; use git_remote_e2ee::crypto::{KeyFile, PublicDevice}; use git_remote_e2ee::policy::DeviceRoles; -use git_remote_e2ee::repository::EncryptedRepository; +use git_remote_e2ee::repository::{EncryptedRepository, RecoveryOptions, RecoveryReport}; use git_remote_e2ee::storage::{FilesystemStorage, GitStorage}; #[derive(Parser)] @@ -15,6 +15,23 @@ struct Cli { command: Command, } +#[derive(Clone, Copy, Debug, ValueEnum)] +enum DeviceRole { + Read, + Write, + Admin, +} + +impl From for DeviceRoles { + fn from(role: DeviceRole) -> Self { + match role { + DeviceRole::Read => Self::reader(), + DeviceRole::Write => Self::collaborator(), + DeviceRole::Admin => Self::owner(), + } + } +} + #[derive(Subcommand)] enum Command { Keygen { @@ -78,8 +95,8 @@ enum Command { key: PathBuf, #[arg(long)] device: PathBuf, - #[arg(long)] - admin: bool, + #[arg(long, value_enum, default_value = "write")] + role: DeviceRole, }, DeviceRevoke { #[arg(long, conflicts_with = "remote", required_unless_present = "remote")] @@ -99,6 +116,26 @@ enum Command { #[arg(long)] key: PathBuf, }, + Recover { + #[arg(long, conflicts_with = "remote", required_unless_present = "remote")] + storage: Option, + #[arg(long, conflicts_with = "storage", required_unless_present = "storage")] + remote: Option, + #[arg(long)] + key: PathBuf, + #[arg(long, default_value = ".")] + repo: PathBuf, + #[arg(long, default_value = "origin")] + remote_name: String, + #[arg(long)] + publish: bool, + #[arg(long, requires = "publish")] + base: Option, + #[arg(long, requires = "publish")] + discard_newer: bool, + #[arg(long, requires = "publish")] + accept_stale_floor: bool, + }, } fn main() -> Result<()> { @@ -178,16 +215,12 @@ fn main() -> Result<()> { remote, key, device, - admin, + role, } => { let pin = admin_pin_path(&key); let public = PublicDevice::read(&device)?; let device_id = public.device_id.clone(); - let roles = if admin { - DeviceRoles::owner() - } else { - DeviceRoles::collaborator() - }; + let roles = DeviceRoles::from(role); let (manifest, policy) = match (storage, remote) { (Some(storage), None) => { EncryptedRepository::new(FilesystemStorage::new(storage), KeyFile::read(&key)?) @@ -239,19 +272,122 @@ fn main() -> Result<()> { }; for device in devices { println!( - "{} reader={} writer={} admin={} status={}", + "{} role={} status={}", device.public.device_id, - device.roles.reader, - device.roles.writer, - device.roles.administrator, + role_name(&device.roles), if device.active() { "active" } else { "revoked" } ); } } + Command::Recover { + storage, + remote, + key, + repo, + remote_name, + publish, + base, + discard_newer, + accept_stale_floor, + } => { + let key = KeyFile::read(&key)?; + let options = RecoveryOptions { + publish, + base: base.as_deref(), + discard_newer, + accept_stale_floor, + }; + let report = match (storage, remote) { + (Some(storage), None) => EncryptedRepository::new( + FilesystemStorage::new(storage), + key, + ) + .recover(&repo, &remote_name, options)?, + (None, Some(remote)) => EncryptedRepository::new(GitStorage::open(&remote)?, key) + .recover(&repo, &remote_name, options)?, + _ => unreachable!("clap requires exactly one backend"), + }; + print_recovery_report(&report); + if publish && let Some(reason) = report.blocked_reason.as_deref() { + bail!("recovery refused: {reason}") + } + } } Ok(()) } +fn print_recovery_report(report: &RecoveryReport) { + println!("{}", format_recovery_report(report)); +} + +fn format_recovery_report(report: &RecoveryReport) -> String { + let mut lines = vec![format!( + "storage head: {} ({})", + report.classification.name(), + report.reason + )]; + if let Some(head) = &report.head_id { + lines.push(format!("head manifest: {head}")); + } + if let Some(signer) = &report.claimed_signer { + if report.classification.name() == "invalid" { + lines.push(format!("claimed signer (signature not verified): {signer}")); + } else { + lines.push(format!("verified signer: {signer}")); + } + } + if let (Some(floor), Some(generation)) = (&report.floor_id, report.floor_generation) { + lines.push(format!( + "continuity floor: {floor} (generation {generation})" + )); + } else { + lines.push( + "continuity floor: unavailable; freshness and fork identity are unverified".to_owned(), + ); + } + for candidate in &report.candidates { + let outer = candidate + .outer_commit + .as_deref() + .unwrap_or("directory backend"); + lines.push(format!( + "candidate: {} generation={} outer={} verified-signer={}", + candidate.manifest_id, candidate.generation, outer, candidate.verified_signer + )); + } + for replay in &report.replays { + lines.push(format!("replay: {replay}")); + } + if report.conflict { + lines.push("conflict: authenticated descendants form a fork".to_owned()); + } + if let Some(base) = &report.default_base { + lines.push(format!("default base: {base}")); + } + if let Some(warning) = &report.warning { + lines.push(format!("warning: {warning}")); + } + if let Some(blocked) = &report.blocked_reason { + lines.push(format!("recovery refused: {blocked}")); + } + if let Some(manifest) = &report.published_manifest { + lines.push(format!("published recovery manifest: {manifest}")); + } + lines.join("\n") +} + +fn role_name(roles: &DeviceRoles) -> &'static str { + if roles.administrator { + "admin" + } else if roles.writer { + "write" + } else if roles.reader { + "read" + } else { + "none" + } +} + fn admin_pin_path(key: &std::path::Path) -> PathBuf { let mut value = key.as_os_str().to_owned(); value.push(".admin-state.json"); @@ -267,3 +403,90 @@ fn open_repository( KeyFile::read(&key)?, )) } + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::{Cli, Command, format_recovery_report}; + use git_remote_e2ee::repository::{RecoveryClass, RecoveryReport}; + + #[test] + fn recover_cli_defaults_to_report_and_accepts_explicit_publication_flags() { + let cli = Cli::try_parse_from([ + "git-e2ee", + "recover", + "--storage", + "/tmp/carrier", + "--key", + "/tmp/key.json", + ]) + .unwrap(); + match cli.command { + Command::Recover { + publish, + remote_name, + accept_stale_floor, + .. + } => { + assert!(!publish); + assert_eq!(remote_name, "origin"); + assert!(!accept_stale_floor); + } + _ => panic!("parsed a different command"), + } + + assert!( + Cli::try_parse_from([ + "git-e2ee", + "recover", + "--storage", + "/tmp/carrier", + "--key", + "/tmp/key.json", + "--discard-newer", + ]) + .is_err() + ); + let cli = Cli::try_parse_from([ + "git-e2ee", + "recover", + "--remote", + "carrier.git", + "--key", + "key.json", + "--publish", + "--base", + "deadbeef", + "--discard-newer", + "--accept-stale-floor", + ]) + .unwrap(); + assert!(matches!( + cli.command, + Command::Recover { publish: true, .. } + )); + } + + #[test] + fn invalid_signer_output_is_labeled_as_a_claim() { + let report = RecoveryReport { + classification: RecoveryClass::Invalid, + reason: "signature verification failed".to_owned(), + head_id: Some("head-id".to_owned()), + claimed_signer: Some("device-id".to_owned()), + floor_id: None, + floor_generation: None, + candidates: Vec::new(), + default_base: None, + replays: Vec::new(), + conflict: false, + freshness_unverified: true, + warning: None, + blocked_reason: None, + published_manifest: None, + }; + let output = format_recovery_report(&report); + assert!(output.contains("claimed signer (signature not verified): device-id")); + } +} diff --git a/src/manifest.rs b/src/manifest.rs index 1692a04..2e354a8 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -7,13 +7,16 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::crypto::{ - KeyFile, SecretKey, SubkeyKind, commit_key, derive_subkey, open_with_key, seal_with_key, - verify_domain, verify_key_commitment, wrap_generation_key, + HPKE_CIPHERTEXT_SIZE, HPKE_ENCAPSULATED_KEY_SIZE, KeyFile, SecretKey, SubkeyKind, commit_key, + derive_subkey, dummy_generation_envelope, open_with_key, random_seed, seal_with_key, + verify_domain, verify_key_commitment, wrap_generation_key_with_seed, }; use crate::policy::PolicyState; -pub const FORMAT_VERSION: u32 = 4; -const MANIFEST_SIGNATURE_DOMAIN: &[u8] = b"git-remote-e2ee manifest v4\0"; +pub const FORMAT_VERSION: u32 = 5; +pub const MAX_GENERATION_ENVELOPES: usize = 4096; +const MIN_GENERATION_ENVELOPES: usize = 4; +const MANIFEST_SIGNATURE_DOMAIN: &[u8] = b"git-remote-e2ee manifest v5\0"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -27,11 +30,20 @@ pub struct PackDescriptor { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct GenerationKeyEnvelope { - pub device_id: String, pub encapsulated_key: String, pub ciphertext: String, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum EnvelopeAuditEntry { + Recipient { + device_id: String, + encapsulation_seed: String, + }, + Dummy, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct Manifest { pub format_version: u32, @@ -45,10 +57,11 @@ pub struct Manifest { pub refs: BTreeMap, pub new_packs: Vec, pub predecessor_key_wrap: Option, + pub introduced_policy: Option>, } impl Manifest { - pub fn genesis(repository_root: String, policy: &PolicyState) -> Self { + pub fn genesis(repository_root: String, policy: &PolicyState, policy_bytes: Vec) -> Self { Self { format_version: FORMAT_VERSION, repository_root, @@ -61,11 +74,13 @@ impl Manifest { refs: BTreeMap::new(), new_packs: Vec::new(), predecessor_key_wrap: None, + introduced_policy: Some(policy_bytes), } } pub fn validate_genesis(&self, policy: &PolicyState) -> Result<()> { validate_manifest_policy(self, policy)?; + validate_introduced_policy(self, policy, true)?; if self.generation != 0 || self.previous.is_some() || self.authorization != ManifestAuthorization::PolicyTransition @@ -109,6 +124,7 @@ impl Manifest { match self.authorization { ManifestAuthorization::Writer => { + validate_introduced_policy(self, selected_policy, false)?; if self.policy_id != previous.policy_id || self.policy_generation != previous.policy_generation { @@ -116,6 +132,7 @@ impl Manifest { } } ManifestAuthorization::PolicyTransition => { + validate_introduced_policy(self, selected_policy, true)?; if self.policy_generation != previous.policy_generation + 1 || selected_policy.body.previous.as_deref() != Some(previous.policy_id.as_str()) { @@ -141,6 +158,8 @@ pub enum ManifestAuthorization { Checkpoint, } +/// The complete plaintext part of a v5 manifest. Device identities and policy +/// metadata are only present in the encrypted sealed header. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ManifestHeader { @@ -148,13 +167,28 @@ pub struct ManifestHeader { pub repository_root: String, pub generation: u64, pub previous: Option, - pub policy_id: String, - pub policy_generation: u64, - pub total_pack_count: u64, pub key_commitment: String, pub generation_key_envelopes: Vec, - pub signer_device_id: String, - pub authorization: ManifestAuthorization, + pub body_digest: String, + pub sealed_header_digest: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SealedHeaderContent { + policy_id: String, + policy_generation: u64, + signer_device_id: String, + authorization: ManifestAuthorization, + total_pack_count: u64, + envelope_audit: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SealedHeaderEnvelope { + content: String, + signature: String, } #[derive(Serialize, Deserialize)] @@ -163,23 +197,37 @@ struct ManifestBody { refs: BTreeMap, new_packs: Vec, predecessor_key_wrap: Option, + introduced_policy: Option, } #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct ManifestEnvelope { header: String, + sealed_header_ciphertext: String, body_ciphertext: String, - signature: String, } -struct ParsedEnvelope { - header: ManifestHeader, +#[derive(Clone, Debug)] +pub struct OpenedManifest { + pub(crate) header: ManifestHeader, + pub(crate) manifest: Manifest, + pub(crate) claimed_signer_id: String, exact_header: Vec, - body_ciphertext: Vec, + exact_sealed_content: Vec, signature: Vec, } +impl OpenedManifest { + pub fn header(&self) -> &ManifestHeader { + &self.header + } + + pub fn manifest(&self) -> &Manifest { + &self.manifest + } +} + pub fn seal_manifest( key: &KeyFile, policy: &PolicyState, @@ -188,153 +236,182 @@ pub fn seal_manifest( manifest: Manifest, ) -> Result> { validate_manifest_policy(&manifest, policy)?; + if manifest.authorization != authorization { + bail!("manifest authorization does not match its transition") + } + match authorization { + ManifestAuthorization::Writer => validate_introduced_policy(&manifest, policy, false)?, + ManifestAuthorization::PolicyTransition if manifest.generation == 0 => { + validate_introduced_policy(&manifest, policy, true)? + } + ManifestAuthorization::PolicyTransition => { + validate_introduced_policy(&manifest, policy, true)? + } + ManifestAuthorization::Checkpoint => { + bail!("checkpoint transitions are reserved but not implemented") + } + } + validate_pack_delta(&manifest)?; + let commitment = commit_key(generation_key); - let mut envelopes = policy - .body - .devices - .iter() - .filter(|device| device.active() && device.roles.reader) - .map(|device| { - let aad = generation_key_aad( - &manifest.repository_root, - manifest.generation, - &manifest.policy_id, - &device.public.device_id, - &commitment, - ); - let (encapsulated, ciphertext) = - wrap_generation_key(&device.public.wrapping_public_key, generation_key, &aad)?; - Ok(GenerationKeyEnvelope { - device_id: device.public.device_id.clone(), - encapsulated_key: BASE64.encode(encapsulated), - ciphertext: BASE64.encode(ciphertext), - }) - }) - .collect::>>()?; - envelopes.sort_by(|a, b| a.device_id.cmp(&b.device_id)); + let (generation_key_envelopes, envelope_audit) = + build_envelopes(policy, &manifest, generation_key, &commitment)?; + let body = serde_json::to_vec(&ManifestBody { + refs: manifest.refs.clone(), + new_packs: manifest.new_packs.clone(), + predecessor_key_wrap: manifest.predecessor_key_wrap.clone(), + introduced_policy: manifest + .introduced_policy + .as_deref() + .map(|bytes| BASE64.encode(bytes)), + })?; + let body_key = derive_subkey( + generation_key, + &manifest.repository_root, + manifest.generation, + SubkeyKind::ManifestBody, + 0, + )?; + let body_ciphertext = seal_with_key( + &body_key, + &body, + &manifest_body_aad(&manifest.repository_root, manifest.generation), + )?; + let body_digest = hex::encode(Sha256::digest(&body_ciphertext)); + let content = SealedHeaderContent { + policy_id: manifest.policy_id.clone(), + policy_generation: manifest.policy_generation, + signer_device_id: key.device_id()?, + authorization, + total_pack_count: manifest.total_pack_count, + envelope_audit, + }; + let exact_sealed_content = serde_json::to_vec(&content)?; let header = ManifestHeader { format_version: manifest.format_version, repository_root: manifest.repository_root.clone(), generation: manifest.generation, previous: manifest.previous.clone(), - policy_id: manifest.policy_id.clone(), - policy_generation: manifest.policy_generation, - total_pack_count: manifest.total_pack_count, key_commitment: commitment, - generation_key_envelopes: envelopes, - signer_device_id: key.device_id()?, - authorization, + generation_key_envelopes, + body_digest, + sealed_header_digest: hex::encode(Sha256::digest(&exact_sealed_content)), }; - validate_recipient_set(&header, policy)?; + validate_public_header(&header)?; + validate_envelope_layout(&header)?; let exact_header = serde_json::to_vec(&header)?; - let body = serde_json::to_vec(&ManifestBody { - refs: manifest.refs, - new_packs: manifest.new_packs, - predecessor_key_wrap: manifest.predecessor_key_wrap, - })?; - let body_key = derive_subkey( + let signed_bytes = + manifest_signed_bytes(&exact_header, &exact_sealed_content, &header.body_digest); + let sealed = SealedHeaderEnvelope { + content: BASE64.encode(&exact_sealed_content), + signature: BASE64.encode(key.sign_domain(MANIFEST_SIGNATURE_DOMAIN, &signed_bytes)?), + }; + let sealed_bytes = serde_json::to_vec(&sealed)?; + let sealed_key = derive_subkey( generation_key, &header.repository_root, header.generation, - SubkeyKind::ManifestBody, + SubkeyKind::SealedHeader, 0, )?; - let body_ciphertext = seal_with_key( - &body_key, - &body, - &manifest_body_aad(&header.repository_root, header.generation), - )?; - let signed = manifest_signed_bytes(&exact_header, &body_ciphertext); - let envelope = ManifestEnvelope { + let sealed_header_ciphertext = seal_with_key(&sealed_key, &sealed_bytes, &exact_header)?; + Ok(serde_json::to_vec(&ManifestEnvelope { header: BASE64.encode(exact_header), + sealed_header_ciphertext: BASE64.encode(sealed_header_ciphertext), body_ciphertext: BASE64.encode(body_ciphertext), - signature: BASE64.encode(key.sign_domain(MANIFEST_SIGNATURE_DOMAIN, &signed)?), - }; - Ok(serde_json::to_vec(&envelope)?) -} - -pub fn read_manifest_header( - encrypted: &[u8], - policy: &PolicyState, - parent_policy: Option<&PolicyState>, -) -> Result { - let parsed = parse_envelope(encrypted)?; - let header = parsed.header; - if header.format_version != FORMAT_VERSION { - bail!("unsupported manifest format {}", header.format_version) - } - if header.repository_root != policy.body.repository_root - || header.policy_id != policy.id - || header.policy_generation != policy.body.generation - { - bail!("manifest header does not match its policy") - } - validate_recipient_set(&header, policy)?; - let signer = match header.authorization { - ManifestAuthorization::Writer => policy - .device(&header.signer_device_id) - .filter(|device| device.active() && device.roles.writer) - .context("manifest was not signed by an active writer")?, - ManifestAuthorization::PolicyTransition => { - let authorizing = parent_policy.unwrap_or(policy); - authorizing - .device(&header.signer_device_id) - .filter(|device| device.active() && device.roles.administrator) - .context("policy transition manifest was not signed by a parent administrator")? - } - ManifestAuthorization::Checkpoint => { - bail!("checkpoint transitions are reserved but not implemented") - } - }; - verify_domain( - &signer.public.signing_public_key, - MANIFEST_SIGNATURE_DOMAIN, - &manifest_signed_bytes(&parsed.exact_header, &parsed.body_ciphertext), - &parsed.signature, - )?; - Ok(header) + })?) } pub fn peek_manifest_header(encrypted: &[u8]) -> Result { - Ok(parse_envelope(encrypted)?.header) + let envelope: ManifestEnvelope = + serde_json::from_slice(encrypted).context("parse manifest envelope")?; + let exact_header = BASE64 + .decode(envelope.header) + .context("decode manifest header")?; + let header: ManifestHeader = + serde_json::from_slice(&exact_header).context("parse manifest header")?; + validate_public_header(&header)?; + Ok(header) } pub fn unwrap_generation_key(header: &ManifestHeader, key: &KeyFile) -> Result { if key.repository_root != header.repository_root { bail!("key file belongs to a different repository") } + validate_public_header(header)?; let device_id = key.device_id()?; - let envelope = header - .generation_key_envelopes - .iter() - .find(|envelope| envelope.device_id == device_id) - .context("device is not an active reader in this generation")?; - let aad = generation_key_aad( + for envelope in &header.generation_key_envelopes { + let Ok(encapsulated_key) = BASE64.decode(&envelope.encapsulated_key) else { + continue; + }; + let Ok(ciphertext) = BASE64.decode(&envelope.ciphertext) else { + continue; + }; + let aad = generation_key_aad( + &header.repository_root, + header.generation, + &device_id, + &header.key_commitment, + ); + if let Ok(generation_key) = key.unwrap_generation_key(&encapsulated_key, &ciphertext, &aad) + && verify_key_commitment(&generation_key, &header.key_commitment).is_ok() + { + return Ok(generation_key); + } + } + bail!( + "device is not an active reader of the current generation (revoked, never added, or envelope malformed)" + ) +} + +pub fn open_manifest(encrypted: &[u8], generation_key: &[u8; 32]) -> Result { + let envelope: ManifestEnvelope = + serde_json::from_slice(encrypted).context("parse manifest envelope")?; + let exact_header = BASE64 + .decode(envelope.header) + .context("decode manifest header")?; + let header: ManifestHeader = + serde_json::from_slice(&exact_header).context("parse manifest header")?; + validate_public_header(&header)?; + verify_key_commitment(generation_key, &header.key_commitment)?; + let sealed_header_ciphertext = BASE64 + .decode(envelope.sealed_header_ciphertext) + .context("decode sealed manifest header")?; + let body_ciphertext = BASE64 + .decode(envelope.body_ciphertext) + .context("decode manifest body")?; + if hex::encode(Sha256::digest(&body_ciphertext)) != header.body_digest { + bail!("manifest body digest mismatch") + } + + let sealed_key = derive_subkey( + generation_key, &header.repository_root, header.generation, - &header.policy_id, - &device_id, - &header.key_commitment, - ); - let generation_key = key.unwrap_generation_key( - &BASE64.decode(&envelope.encapsulated_key)?, - &BASE64.decode(&envelope.ciphertext)?, - &aad, + SubkeyKind::SealedHeader, + 0, )?; - verify_key_commitment(&generation_key, &header.key_commitment)?; - Ok(generation_key) -} + let sealed_bytes = open_with_key(&sealed_key, &sealed_header_ciphertext, &exact_header) + .context("open sealed manifest header")?; + let sealed: SealedHeaderEnvelope = + serde_json::from_slice(&sealed_bytes).context("parse sealed manifest header")?; + let exact_sealed_content = BASE64 + .decode(sealed.content) + .context("decode sealed manifest content")?; + if hex::encode(Sha256::digest(&exact_sealed_content)) != header.sealed_header_digest { + bail!("sealed manifest header digest mismatch") + } + let content: SealedHeaderContent = + serde_json::from_slice(&exact_sealed_content).context("parse sealed manifest content")?; + if serde_json::to_vec(&content)? != exact_sealed_content { + bail!("sealed manifest content is not canonically encoded") + } + let signature = BASE64 + .decode(sealed.signature) + .context("decode manifest signature")?; + let claimed_signer_id = content.signer_device_id.clone(); -pub fn open_manifest( - encrypted: &[u8], - policy: &PolicyState, - parent_policy: Option<&PolicyState>, - generation_key: &[u8; 32], -) -> Result { - let header = read_manifest_header(encrypted, policy, parent_policy)?; - verify_key_commitment(generation_key, &header.key_commitment)?; - let parsed = parse_envelope(encrypted)?; let body_key = derive_subkey( generation_key, &header.repository_root, @@ -344,28 +421,112 @@ pub fn open_manifest( )?; let body_bytes = open_with_key( &body_key, - &parsed.body_ciphertext, + &body_ciphertext, &manifest_body_aad(&header.repository_root, header.generation), )?; let body: ManifestBody = serde_json::from_slice(&body_bytes).context("parse manifest body")?; + let introduced_policy = body + .introduced_policy + .map(|encoded| BASE64.decode(encoded).context("decode introduced policy")) + .transpose()?; let manifest = Manifest { format_version: header.format_version, - repository_root: header.repository_root, + repository_root: header.repository_root.clone(), generation: header.generation, - previous: header.previous, - policy_id: header.policy_id, - policy_generation: header.policy_generation, - authorization: header.authorization, - total_pack_count: header.total_pack_count, + previous: header.previous.clone(), + policy_id: content.policy_id, + policy_generation: content.policy_generation, + authorization: content.authorization, + total_pack_count: content.total_pack_count, refs: body.refs, new_packs: body.new_packs, predecessor_key_wrap: body.predecessor_key_wrap, + introduced_policy, }; validate_pack_delta(&manifest)?; if (manifest.generation == 0) != manifest.predecessor_key_wrap.is_none() { bail!("manifest predecessor key link does not match its generation") } - Ok(manifest) + if manifest.generation == 0 && manifest.previous.is_some() + || manifest.generation > 0 && manifest.previous.is_none() + { + bail!("manifest previous pointer does not match its generation") + } + Ok(OpenedManifest { + header, + manifest, + claimed_signer_id, + exact_header, + exact_sealed_content, + signature, + }) +} + +pub fn verify_manifest( + opened: &OpenedManifest, + selected_policy: &PolicyState, + parent_policy: Option<&PolicyState>, + generation_key: &[u8; 32], +) -> Result<()> { + let manifest = &opened.manifest; + if manifest.format_version != FORMAT_VERSION + || manifest.repository_root != selected_policy.body.repository_root + || manifest.policy_id != selected_policy.id + || manifest.policy_generation != selected_policy.body.generation + { + bail!("manifest does not match policy state") + } + validate_public_header(&opened.header)?; + validate_envelope_layout(&opened.header)?; + if manifest.format_version != opened.header.format_version + || manifest.repository_root != opened.header.repository_root + || manifest.generation != opened.header.generation + || manifest.previous != opened.header.previous + { + bail!("sealed manifest metadata does not match its plaintext header") + } + let content: SealedHeaderContent = serde_json::from_slice(&opened.exact_sealed_content) + .context("parse sealed manifest content")?; + if serde_json::to_vec(&content)? != opened.exact_sealed_content + || content.policy_id != manifest.policy_id + || content.policy_generation != manifest.policy_generation + || content.authorization != manifest.authorization + || content.total_pack_count != manifest.total_pack_count + { + bail!("opened manifest fields do not match the signed sealed header") + } + let authorizing = match content.authorization { + ManifestAuthorization::Writer => selected_policy + .device(&content.signer_device_id) + .filter(|device| device.active() && device.roles.writer) + .context("manifest was not signed by an active writer")?, + ManifestAuthorization::PolicyTransition => { + let authorizing = parent_policy.unwrap_or(selected_policy); + authorizing + .device(&content.signer_device_id) + .filter(|device| device.active() && device.roles.administrator) + .context("policy transition manifest was not signed by a parent administrator")? + } + ManifestAuthorization::Checkpoint => { + bail!("checkpoint transitions are reserved but not implemented") + } + }; + verify_domain( + &authorizing.public.signing_public_key, + MANIFEST_SIGNATURE_DOMAIN, + &manifest_signed_bytes( + &opened.exact_header, + &opened.exact_sealed_content, + &opened.header.body_digest, + ), + &opened.signature, + )?; + validate_envelope_audit( + &opened.header, + &content.envelope_audit, + selected_policy, + generation_key, + ) } pub fn wrap_predecessor_key( @@ -429,27 +590,146 @@ pub fn unwrap_predecessor_key( } pub fn pack_aad(repository_root: &str, generation: u64, ordinal: u64) -> Vec { - let mut aad = b"git-remote-e2ee pack v4\0".to_vec(); + let mut aad = b"git-remote-e2ee pack v5\0".to_vec(); push_field(&mut aad, repository_root.as_bytes()); aad.extend_from_slice(&generation.to_le_bytes()); aad.extend_from_slice(&ordinal.to_le_bytes()); aad } -fn parse_envelope(encrypted: &[u8]) -> Result { - let envelope: ManifestEnvelope = - serde_json::from_slice(encrypted).context("parse manifest envelope")?; - let exact_header = BASE64 - .decode(envelope.header) - .context("decode manifest header")?; - let header: ManifestHeader = - serde_json::from_slice(&exact_header).context("parse manifest header")?; - Ok(ParsedEnvelope { - header, - exact_header, - body_ciphertext: BASE64.decode(envelope.body_ciphertext)?, - signature: BASE64.decode(envelope.signature)?, - }) +fn build_envelopes( + policy: &PolicyState, + manifest: &Manifest, + generation_key: &[u8; 32], + commitment: &str, +) -> Result<(Vec, Vec)> { + let readers: Vec<_> = policy + .body + .devices + .iter() + .filter(|device| device.active() && device.roles.reader) + .collect(); + if readers.is_empty() { + bail!("policy has no active readers") + } + let count = readers + .len() + .checked_next_power_of_two() + .context("active reader count is too large")? + .max(MIN_GENERATION_ENVELOPES); + if count > MAX_GENERATION_ENVELOPES { + bail!("padded generation envelope count exceeds {MAX_GENERATION_ENVELOPES}") + } + + let mut pairs = Vec::with_capacity(count); + for device in readers { + let seed = random_seed(); + let aad = generation_key_aad( + &manifest.repository_root, + manifest.generation, + &device.public.device_id, + commitment, + ); + let (encapsulated, ciphertext) = wrap_generation_key_with_seed( + &device.public.wrapping_public_key, + generation_key, + &aad, + &seed, + )?; + pairs.push(( + encapsulated.clone(), + GenerationKeyEnvelope { + encapsulated_key: BASE64.encode(&encapsulated), + ciphertext: BASE64.encode(ciphertext), + }, + EnvelopeAuditEntry::Recipient { + device_id: device.public.device_id.clone(), + encapsulation_seed: BASE64.encode(seed), + }, + )); + } + while pairs.len() < count { + let (encapsulated, ciphertext) = dummy_generation_envelope()?; + pairs.push(( + encapsulated.clone(), + GenerationKeyEnvelope { + encapsulated_key: BASE64.encode(encapsulated), + ciphertext: BASE64.encode(ciphertext), + }, + EnvelopeAuditEntry::Dummy, + )); + } + pairs.sort_by(|left, right| left.0.cmp(&right.0)); + let mut envelopes = Vec::with_capacity(count); + let mut audit = Vec::with_capacity(count); + for (_, envelope, entry) in pairs { + envelopes.push(envelope); + audit.push(entry); + } + Ok((envelopes, audit)) +} + +fn validate_envelope_audit( + header: &ManifestHeader, + audit: &[EnvelopeAuditEntry], + policy: &PolicyState, + generation_key: &[u8; 32], +) -> Result<()> { + if audit.len() != header.generation_key_envelopes.len() { + bail!("envelope audit count does not match the envelope list") + } + let active: HashSet<_> = policy + .body + .devices + .iter() + .filter(|device| device.active() && device.roles.reader) + .map(|device| device.public.device_id.as_str()) + .collect(); + let mut audited = HashSet::with_capacity(audit.len()); + for (envelope, entry) in header.generation_key_envelopes.iter().zip(audit) { + let EnvelopeAuditEntry::Recipient { + device_id, + encapsulation_seed, + } = entry + else { + continue; + }; + if !active.contains(device_id.as_str()) { + bail!("envelope audit names a device that is not an active reader") + } + if !audited.insert(device_id.as_str()) { + bail!("envelope audit contains a duplicate reader") + } + let seed: [u8; 32] = BASE64 + .decode(encapsulation_seed) + .context("decode envelope audit seed")? + .try_into() + .map_err(|_| anyhow::anyhow!("invalid envelope audit seed length"))?; + let recipient = policy + .device(device_id) + .context("envelope audit names an unknown device")?; + let aad = generation_key_aad( + &header.repository_root, + header.generation, + device_id, + &header.key_commitment, + ); + let (encapsulated, ciphertext) = wrap_generation_key_with_seed( + &recipient.public.wrapping_public_key, + generation_key, + &aad, + &seed, + )?; + if BASE64.encode(encapsulated) != envelope.encapsulated_key + || BASE64.encode(ciphertext) != envelope.ciphertext + { + bail!("generation envelope does not match its sealed audit entry") + } + } + if audited != active { + bail!("envelope audit must match active readers exactly") + } + Ok(()) } fn validate_manifest_policy(manifest: &Manifest, policy: &PolicyState) -> Result<()> { @@ -463,21 +743,88 @@ fn validate_manifest_policy(manifest: &Manifest, policy: &PolicyState) -> Result Ok(()) } -fn validate_recipient_set(header: &ManifestHeader, policy: &PolicyState) -> Result<()> { - let active: HashSet<_> = policy - .body - .devices - .iter() - .filter(|device| device.active() && device.roles.reader) - .map(|device| device.public.device_id.as_str()) - .collect(); - let wrapped: HashSet<_> = header - .generation_key_envelopes - .iter() - .map(|envelope| envelope.device_id.as_str()) - .collect(); - if wrapped.len() != header.generation_key_envelopes.len() || wrapped != active { - bail!("generation key envelopes must match active readers exactly") +fn validate_introduced_policy( + manifest: &Manifest, + selected_policy: &PolicyState, + should_introduce: bool, +) -> Result<()> { + match (&manifest.introduced_policy, should_introduce) { + (Some(bytes), true) => { + let policy = PolicyState::parse(bytes).context("parse introduced policy")?; + if policy.id != selected_policy.id { + bail!("manifest introduced policy id does not match its sealed header") + } + } + (None, false) => {} + (Some(_), false) => bail!("ordinary manifest unexpectedly introduces a policy"), + (None, true) => bail!("policy transition manifest is missing its introduced policy"), + } + Ok(()) +} + +fn validate_public_header(header: &ManifestHeader) -> Result<()> { + if header.format_version != FORMAT_VERSION { + bail!("unsupported manifest format {}", header.format_version) + } + if header.repository_root.len() != 64 + || !header + .repository_root + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + bail!("invalid manifest repository root") + } + if header.key_commitment.len() != 64 + || !header + .key_commitment + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + bail!("invalid generation key commitment") + } + if header.body_digest.len() != 64 + || !header + .body_digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + || header.sealed_header_digest.len() != 64 + || !header + .sealed_header_digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + bail!("invalid manifest part digest") + } + let count = header.generation_key_envelopes.len(); + if !(MIN_GENERATION_ENVELOPES..=MAX_GENERATION_ENVELOPES).contains(&count) + || !count.is_power_of_two() + { + bail!("invalid padded generation envelope count") + } + Ok(()) +} + +fn validate_envelope_layout(header: &ManifestHeader) -> Result<()> { + let mut previous: Option> = None; + for envelope in &header.generation_key_envelopes { + let encapsulated = BASE64 + .decode(&envelope.encapsulated_key) + .context("decode HPKE encapsulated key")?; + let ciphertext = BASE64 + .decode(&envelope.ciphertext) + .context("decode HPKE ciphertext")?; + if encapsulated.len() != HPKE_ENCAPSULATED_KEY_SIZE + || ciphertext.len() != HPKE_CIPHERTEXT_SIZE + { + bail!("invalid generation envelope length") + } + if previous + .as_ref() + .is_some_and(|prior| prior >= &encapsulated) + { + bail!("generation envelopes are not sorted by encapsulated-key bytes") + } + previous = Some(encapsulated); } Ok(()) } @@ -498,38 +845,38 @@ fn validate_pack_delta(manifest: &Manifest) -> Result<()> { Ok(()) } -fn manifest_signed_bytes(exact_header: &[u8], body_ciphertext: &[u8]) -> Vec { - let mut bytes = Vec::with_capacity(exact_header.len() + 32); - bytes.extend_from_slice(exact_header); - bytes.extend_from_slice(&Sha256::digest(body_ciphertext)); +fn manifest_signed_bytes(exact_header: &[u8], exact_content: &[u8], body_digest: &str) -> Vec { + let mut bytes = Vec::with_capacity(exact_header.len() + exact_content.len() + 72); + push_field(&mut bytes, exact_header); + push_field(&mut bytes, exact_content); + push_field(&mut bytes, body_digest.as_bytes()); bytes } fn generation_key_aad( root: &str, generation: u64, - policy_id: &str, recipient_id: &str, commitment: &str, ) -> Vec { - let mut aad = b"git-remote-e2ee generation envelope v4\0".to_vec(); + let mut aad = b"git-remote-e2ee generation envelope v5\0".to_vec(); push_field(&mut aad, root.as_bytes()); + aad.extend_from_slice(&FORMAT_VERSION.to_le_bytes()); aad.extend_from_slice(&generation.to_le_bytes()); - push_field(&mut aad, policy_id.as_bytes()); - push_field(&mut aad, recipient_id.as_bytes()); push_field(&mut aad, commitment.as_bytes()); + push_field(&mut aad, recipient_id.as_bytes()); aad } fn manifest_body_aad(root: &str, generation: u64) -> Vec { - let mut aad = b"git-remote-e2ee manifest body v4\0".to_vec(); + let mut aad = b"git-remote-e2ee manifest body v5\0".to_vec(); push_field(&mut aad, root.as_bytes()); aad.extend_from_slice(&generation.to_le_bytes()); aad } fn predecessor_link_aad(root: &str, generation: u64, previous_id: &str) -> Vec { - let mut aad = b"git-remote-e2ee predecessor link v4\0".to_vec(); + let mut aad = b"git-remote-e2ee predecessor link v5\0".to_vec(); push_field(&mut aad, root.as_bytes()); aad.extend_from_slice(&generation.to_le_bytes()); push_field(&mut aad, previous_id.as_bytes()); @@ -544,7 +891,7 @@ fn push_field(output: &mut Vec, value: &[u8]) { #[cfg(test)] mod tests { use super::*; - use crate::crypto::{PublicDevice, random_key}; + use crate::crypto::{KeyFile, open_with_key, random_key}; use crate::policy::{DeviceRecord, DeviceRoles}; fn add_reader(owner: &KeyFile, parent: &PolicyState, reader: &KeyFile) -> PolicyState { @@ -559,12 +906,156 @@ mod tests { policy } + fn manifest_for( + policy: &PolicyState, + generation: u64, + auth: ManifestAuthorization, + ) -> Manifest { + Manifest { + format_version: FORMAT_VERSION, + repository_root: policy.body.repository_root.clone(), + generation, + previous: (generation > 0).then(|| "11".repeat(32)), + policy_id: policy.id.clone(), + policy_generation: policy.body.generation, + authorization: auth, + total_pack_count: 0, + refs: BTreeMap::new(), + new_packs: Vec::new(), + predecessor_key_wrap: (generation > 0).then(|| BASE64.encode([7_u8; 64])), + introduced_policy: None, + } + } + + fn rewrite_signed_manifest( + encrypted: &[u8], + signer: &KeyFile, + generation_key: &[u8; 32], + edit: impl FnOnce(&mut ManifestHeader, &mut SealedHeaderContent), + ) -> Vec { + let mut envelope: ManifestEnvelope = serde_json::from_slice(encrypted).unwrap(); + let mut header_bytes = BASE64.decode(&envelope.header).unwrap(); + let mut header: ManifestHeader = serde_json::from_slice(&header_bytes).unwrap(); + let sealed_ciphertext = BASE64.decode(&envelope.sealed_header_ciphertext).unwrap(); + let sealed_key = derive_subkey( + generation_key, + &header.repository_root, + header.generation, + SubkeyKind::SealedHeader, + 0, + ) + .unwrap(); + let sealed_bytes = open_with_key(&sealed_key, &sealed_ciphertext, &header_bytes).unwrap(); + let mut sealed: SealedHeaderEnvelope = serde_json::from_slice(&sealed_bytes).unwrap(); + let mut content: SealedHeaderContent = + serde_json::from_slice(&BASE64.decode(&sealed.content).unwrap()).unwrap(); + edit(&mut header, &mut content); + let exact_content = serde_json::to_vec(&content).unwrap(); + header.sealed_header_digest = hex::encode(Sha256::digest(&exact_content)); + header_bytes = serde_json::to_vec(&header).unwrap(); + let body_ciphertext = BASE64.decode(&envelope.body_ciphertext).unwrap(); + let signed = manifest_signed_bytes(&header_bytes, &exact_content, &header.body_digest); + sealed.content = BASE64.encode(&exact_content); + sealed.signature = BASE64.encode( + signer + .sign_domain(MANIFEST_SIGNATURE_DOMAIN, &signed) + .unwrap(), + ); + let sealed_bytes = serde_json::to_vec(&sealed).unwrap(); + envelope.header = BASE64.encode(&header_bytes); + envelope.sealed_header_ciphertext = + BASE64.encode(seal_with_key(&sealed_key, &sealed_bytes, &header_bytes).unwrap()); + envelope.body_ciphertext = BASE64.encode(body_ciphertext); + serde_json::to_vec(&envelope).unwrap() + } + + fn writer_manifest( + signer: &KeyFile, + policy: &PolicyState, + generation_key: &[u8; 32], + ) -> Vec { + let manifest = manifest_for(policy, 2, ManifestAuthorization::Writer); + seal_manifest( + signer, + policy, + generation_key, + ManifestAuthorization::Writer, + manifest, + ) + .unwrap() + } + + #[test] + fn anonymous_envelopes_are_padded_and_open_by_trial_decryption() { + let owner = KeyFile::generate(); + let reader_a = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let reader_b = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let (genesis, genesis_bytes) = PolicyState::genesis(&owner).unwrap(); + let mut devices = genesis.body.devices.clone(); + for reader in [&reader_a, &reader_b] { + devices.push(DeviceRecord { + public: reader.public_device().unwrap(), + roles: DeviceRoles::collaborator(), + revoked_at: None, + }); + } + let (policy, _) = PolicyState::successor(&genesis, devices, &owner).unwrap(); + let (policy5, _policy5_bytes) = + PolicyState::successor(&policy, policy.body.devices.clone(), &owner).unwrap(); + let mut one_reader = manifest_for(&genesis, 0, ManifestAuthorization::PolicyTransition); + one_reader.introduced_policy = Some(genesis_bytes); + let bytes = seal_manifest( + &owner, + &genesis, + &random_key(), + ManifestAuthorization::PolicyTransition, + one_reader, + ) + .unwrap(); + let header = peek_manifest_header(&bytes).unwrap(); + assert_eq!(header.generation_key_envelopes.len(), 4); + let key = unwrap_generation_key(&header, &owner).unwrap(); + let opened = open_manifest(&bytes, &key).unwrap(); + verify_manifest(&opened, &genesis, None, &key).unwrap(); + + let mut five_readers_devices = policy5.body.devices.clone(); + for _ in 0..2 { + let extra = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + five_readers_devices.push(DeviceRecord { + public: extra.public_device().unwrap(), + roles: DeviceRoles::collaborator(), + revoked_at: None, + }); + } + let (five_readers, five_readers_bytes) = + PolicyState::successor(&policy5, five_readers_devices, &owner).unwrap(); + let mut many = manifest_for(&five_readers, 4, ManifestAuthorization::PolicyTransition); + many.previous = Some("22".repeat(32)); + many.predecessor_key_wrap = Some(BASE64.encode([8_u8; 64])); + many.introduced_policy = Some(five_readers_bytes); + let many_bytes = seal_manifest( + &owner, + &five_readers, + &random_key(), + ManifestAuthorization::PolicyTransition, + many, + ) + .unwrap(); + assert_eq!( + peek_manifest_header(&many_bytes) + .unwrap() + .generation_key_envelopes + .len(), + 8 + ); + } + #[test] fn predecessor_link_must_match_parent_commitment() { let owner = KeyFile::generate(); - let (policy, _) = PolicyState::genesis(&owner).unwrap(); + let (policy, policy_bytes) = PolicyState::genesis(&owner).unwrap(); let parent_key = random_key(); - let parent = Manifest::genesis(owner.repository_root.clone(), &policy); + let parent = Manifest::genesis(owner.repository_root.clone(), &policy, policy_bytes); let parent_bytes = seal_manifest( &owner, &policy, @@ -574,32 +1065,22 @@ mod tests { ) .unwrap(); let parent_id = crate::crypto::object_id(&parent_bytes); - let parent_header = read_manifest_header(&parent_bytes, &policy, None).unwrap(); + let parent_header = peek_manifest_header(&parent_bytes).unwrap(); let child_key = random_key(); let wrong_parent_key = random_key(); - let child = Manifest { - format_version: FORMAT_VERSION, - repository_root: owner.repository_root.clone(), - generation: 1, - previous: Some(parent_id.clone()), - policy_id: policy.id.clone(), - policy_generation: policy.body.generation, - authorization: ManifestAuthorization::Writer, - total_pack_count: 0, - refs: BTreeMap::new(), - new_packs: Vec::new(), - predecessor_key_wrap: Some( - wrap_predecessor_key( - &child_key, - &wrong_parent_key, - &owner.repository_root, - 1, - &parent_id, - ) - .unwrap(), - ), - }; + let mut child = manifest_for(&policy, 1, ManifestAuthorization::Writer); + child.previous = Some(parent_id.clone()); + child.predecessor_key_wrap = Some( + wrap_predecessor_key( + &child_key, + &wrong_parent_key, + &owner.repository_root, + 1, + &parent_id, + ) + .unwrap(), + ); let child_bytes = seal_manifest( &owner, &policy, @@ -608,186 +1089,244 @@ mod tests { child, ) .unwrap(); - let opened = open_manifest(&child_bytes, &policy, None, &child_key).unwrap(); - assert!(unwrap_predecessor_key(&child_key, &opened, &parent_header).is_err()); + let opened = open_manifest(&child_bytes, &child_key).unwrap(); + assert!(unwrap_predecessor_key(&child_key, &opened.manifest, &parent_header).is_err()); } #[test] - fn every_validator_rejects_missing_or_duplicate_recipient_envelopes() { + fn malicious_writer_cannot_replace_a_reader_envelope_with_garbage() { let owner = KeyFile::generate(); let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); - let (parent, _) = PolicyState::genesis(&owner).unwrap(); - let policy = add_reader(&owner, &parent, &reader); + let (genesis, _) = PolicyState::genesis(&owner).unwrap(); + let policy = add_reader(&owner, &genesis, &reader); let generation_key = random_key(); - let manifest = Manifest { - format_version: FORMAT_VERSION, - repository_root: owner.repository_root.clone(), - generation: 1, - previous: Some("00".repeat(32)), - policy_id: policy.id.clone(), - policy_generation: policy.body.generation, - authorization: ManifestAuthorization::PolicyTransition, - total_pack_count: 0, - refs: BTreeMap::new(), - new_packs: Vec::new(), - predecessor_key_wrap: Some(BASE64.encode([0_u8; 64])), - }; - let bytes = seal_manifest( - &owner, - &policy, + let valid = writer_manifest(&reader, &policy, &generation_key); + let corrupted = rewrite_signed_manifest( + &valid, + &reader, &generation_key, - ManifestAuthorization::PolicyTransition, - manifest, - ) - .unwrap(); - let mut outer: ManifestEnvelope = serde_json::from_slice(&bytes).unwrap(); - let mut header: ManifestHeader = - serde_json::from_slice(&BASE64.decode(&outer.header).unwrap()).unwrap(); - header.generation_key_envelopes.pop(); - outer.header = BASE64.encode(serde_json::to_vec(&header).unwrap()); - let missing = serde_json::to_vec(&outer).unwrap(); - assert!(read_manifest_header(&missing, &policy, Some(&parent)).is_err()); - - let mut outer: ManifestEnvelope = serde_json::from_slice(&bytes).unwrap(); - let mut header: ManifestHeader = - serde_json::from_slice(&BASE64.decode(&outer.header).unwrap()).unwrap(); - header - .generation_key_envelopes - .push(header.generation_key_envelopes[0].clone()); - outer.header = BASE64.encode(serde_json::to_vec(&header).unwrap()); - let duplicate = serde_json::to_vec(&outer).unwrap(); - assert!(read_manifest_header(&duplicate, &policy, Some(&parent)).is_err()); + |header, audit| { + let index = audit + .envelope_audit + .iter() + .position(|entry| matches!(entry, EnvelopeAuditEntry::Recipient { device_id, .. } if device_id == &reader.device_id().unwrap())) + .unwrap(); + header.generation_key_envelopes[index].ciphertext = + BASE64.encode([0_u8; HPKE_CIPHERTEXT_SIZE]); + }, + ); + let header = peek_manifest_header(&corrupted).unwrap(); + let owner_key = unwrap_generation_key(&header, &owner).unwrap(); + assert_eq!(*owner_key, *generation_key); + let opened = open_manifest(&corrupted, &owner_key).unwrap(); + let error = verify_manifest(&opened, &policy, None, &owner_key) + .unwrap_err() + .to_string(); + assert!(error.contains("does not match its sealed audit entry")); } #[test] - fn signed_extra_envelope_for_revoked_reader_is_rejected() { + fn reader_only_device_cannot_sign_an_ordinary_manifest() { let owner = KeyFile::generate(); let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); let (genesis, _) = PolicyState::genesis(&owner).unwrap(); - let with_reader = add_reader(&owner, &genesis, &reader); - let mut devices = with_reader.body.devices.clone(); - devices - .iter_mut() - .find(|device| device.public.device_id == reader.device_id().unwrap()) - .unwrap() - .revoked_at = Some(with_reader.body.generation + 1); - let (revoked, _) = PolicyState::successor(&with_reader, devices, &owner).unwrap(); - revoked.validate_successor(&with_reader).unwrap(); - - let generation_key = random_key(); - let manifest = Manifest { - format_version: FORMAT_VERSION, - repository_root: owner.repository_root.clone(), - generation: 2, - previous: Some("00".repeat(32)), - policy_id: revoked.id.clone(), - policy_generation: revoked.body.generation, - authorization: ManifestAuthorization::PolicyTransition, - total_pack_count: 0, - refs: BTreeMap::new(), - new_packs: Vec::new(), - predecessor_key_wrap: Some(BASE64.encode([0_u8; 64])), - }; - let bytes = seal_manifest( - &owner, - &revoked, - &generation_key, - ManifestAuthorization::PolicyTransition, - manifest, - ) - .unwrap(); - let mut outer: ManifestEnvelope = serde_json::from_slice(&bytes).unwrap(); - let mut header: ManifestHeader = - serde_json::from_slice(&BASE64.decode(&outer.header).unwrap()).unwrap(); - let public = reader.public_device().unwrap(); - let aad = generation_key_aad( - &header.repository_root, - header.generation, - &header.policy_id, - &public.device_id, - &header.key_commitment, - ); - let (encapsulated, ciphertext) = - wrap_generation_key(&public.wrapping_public_key, &generation_key, &aad).unwrap(); - header.generation_key_envelopes.push(GenerationKeyEnvelope { - device_id: public.device_id, - encapsulated_key: BASE64.encode(encapsulated), - ciphertext: BASE64.encode(ciphertext), + let mut devices = genesis.body.devices.clone(); + devices.push(DeviceRecord { + public: reader.public_device().unwrap(), + roles: DeviceRoles::reader(), + revoked_at: None, }); - header - .generation_key_envelopes - .sort_by(|a, b| a.device_id.cmp(&b.device_id)); - let exact_header = serde_json::to_vec(&header).unwrap(); - let body_ciphertext = BASE64.decode(&outer.body_ciphertext).unwrap(); - outer.header = BASE64.encode(&exact_header); - outer.signature = BASE64.encode( - owner - .sign_domain( - MANIFEST_SIGNATURE_DOMAIN, - &manifest_signed_bytes(&exact_header, &body_ciphertext), - ) - .unwrap(), - ); - let malicious = serde_json::to_vec(&outer).unwrap(); - let error = read_manifest_header(&malicious, &revoked, Some(&with_reader)) + let (policy, _) = PolicyState::successor(&genesis, devices, &owner).unwrap(); + let generation_key = random_key(); + + // seal_manifest can construct a cryptographically valid signature from + // any key. Verification must still enforce the signer's policy role. + let encrypted = writer_manifest(&reader, &policy, &generation_key); + let opened = open_manifest(&encrypted, &generation_key).unwrap(); + let error = verify_manifest(&opened, &policy, None, &generation_key) .unwrap_err() .to_string(); - assert!( - error.contains("must match active readers exactly"), - "unexpected error: {error}" - ); + assert!(error.contains("manifest was not signed by an active writer")); } #[test] - fn wrong_recipient_key_is_rejected_by_signed_commitment() { + fn reader_trials_all_envelopes_and_reports_inactive_membership_clearly() { let owner = KeyFile::generate(); - let (policy, _) = PolicyState::genesis(&owner).unwrap(); - let intended = random_key(); - let wrong = random_key(); - let manifest = Manifest::genesis(owner.repository_root.clone(), &policy); - let bytes = seal_manifest( + let outsider = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let (policy, bytes) = PolicyState::genesis(&owner).unwrap(); + let manifest = Manifest::genesis(owner.repository_root.clone(), &policy, bytes); + let encrypted = seal_manifest( &owner, &policy, - &intended, + &random_key(), ManifestAuthorization::PolicyTransition, manifest, ) .unwrap(); - let mut outer: ManifestEnvelope = serde_json::from_slice(&bytes).unwrap(); - let mut header: ManifestHeader = - serde_json::from_slice(&BASE64.decode(&outer.header).unwrap()).unwrap(); - let PublicDevice { - wrapping_public_key, - device_id, - .. - } = owner.public_device().unwrap(); - let aad = generation_key_aad( - &header.repository_root, - header.generation, - &header.policy_id, - &device_id, - &header.key_commitment, + let error = unwrap_generation_key(&peek_manifest_header(&encrypted).unwrap(), &outsider) + .unwrap_err() + .to_string(); + assert!(error.contains("not an active reader of the current generation")); + } + + #[test] + fn head_envelope_list_is_bounded_before_trial_decryption() { + let owner = KeyFile::generate(); + let header = ManifestHeader { + format_version: FORMAT_VERSION, + repository_root: owner.repository_root.clone(), + generation: 0, + previous: None, + key_commitment: "00".repeat(32), + generation_key_envelopes: vec![ + GenerationKeyEnvelope { + encapsulated_key: String::new(), + ciphertext: String::new(), + }; + MAX_GENERATION_ENVELOPES * 2 + ], + body_digest: "11".repeat(32), + sealed_header_digest: "22".repeat(32), + }; + let error = unwrap_generation_key(&header, &owner) + .unwrap_err() + .to_string(); + assert!(error.contains("invalid padded generation envelope count")); + } + + #[test] + fn audit_rejects_missing_duplicate_extra_and_revoked_reader_entries() { + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let (parent, _) = PolicyState::genesis(&owner).unwrap(); + let policy = add_reader(&owner, &parent, &reader); + let generation_key = random_key(); + let valid = writer_manifest(&owner, &policy, &generation_key); + let (real_indices, dummy_index) = { + let opened = open_manifest(&valid, &generation_key).unwrap(); + let outer: ManifestEnvelope = serde_json::from_slice(&valid).unwrap(); + let header: ManifestHeader = + serde_json::from_slice(&BASE64.decode(outer.header).unwrap()).unwrap(); + let sealed_ciphertext = BASE64.decode(outer.sealed_header_ciphertext).unwrap(); + let key = derive_subkey( + &generation_key, + &header.repository_root, + header.generation, + SubkeyKind::SealedHeader, + 0, + ) + .unwrap(); + let opened_sealed = open_with_key( + &key, + &sealed_ciphertext, + &serde_json::to_vec(&header).unwrap(), + ) + .unwrap(); + let sealed: SealedHeaderEnvelope = serde_json::from_slice(&opened_sealed).unwrap(); + let content: SealedHeaderContent = + serde_json::from_slice(&BASE64.decode(sealed.content).unwrap()).unwrap(); + let real = content + .envelope_audit + .iter() + .enumerate() + .filter(|(_, entry)| matches!(entry, EnvelopeAuditEntry::Recipient { .. })) + .map(|(index, _)| index) + .collect::>(); + let dummy = content + .envelope_audit + .iter() + .position(|entry| matches!(entry, EnvelopeAuditEntry::Dummy)) + .unwrap(); + assert_eq!(opened.manifest.policy_id, policy.id); + (real, dummy) + }; + let missing = rewrite_signed_manifest(&valid, &owner, &generation_key, |_, audit| { + audit.envelope_audit.pop(); + }); + assert!( + verify_manifest( + &open_manifest(&missing, &generation_key).unwrap(), + &policy, + None, + &generation_key + ) + .is_err() ); - let (encapsulated, ciphertext) = - wrap_generation_key(&wrapping_public_key, &wrong, &aad).unwrap(); - header.generation_key_envelopes = vec![GenerationKeyEnvelope { - device_id, - encapsulated_key: BASE64.encode(encapsulated), - ciphertext: BASE64.encode(ciphertext), - }]; - let exact_header = serde_json::to_vec(&header).unwrap(); - let body_ciphertext = BASE64.decode(&outer.body_ciphertext).unwrap(); - outer.header = BASE64.encode(&exact_header); - outer.signature = BASE64.encode( - owner - .sign_domain( - MANIFEST_SIGNATURE_DOMAIN, - &manifest_signed_bytes(&exact_header, &body_ciphertext), - ) - .unwrap(), + + let duplicate = rewrite_signed_manifest(&valid, &owner, &generation_key, |_, audit| { + let duplicate_entry = audit.envelope_audit[real_indices[0]].clone(); + audit.envelope_audit[real_indices[1]] = duplicate_entry; + }); + assert!( + verify_manifest( + &open_manifest(&duplicate, &generation_key).unwrap(), + &policy, + None, + &generation_key + ) + .is_err() + ); + + let extra = rewrite_signed_manifest(&valid, &owner, &generation_key, |_, audit| { + audit.envelope_audit[dummy_index] = EnvelopeAuditEntry::Recipient { + device_id: "ff".repeat(32), + encapsulation_seed: BASE64.encode([0_u8; 32]), + }; + }); + assert!( + verify_manifest( + &open_manifest(&extra, &generation_key).unwrap(), + &policy, + None, + &generation_key + ) + .is_err() + ); + + let wrong_seed = rewrite_signed_manifest(&valid, &owner, &generation_key, |_, audit| { + let EnvelopeAuditEntry::Recipient { + encapsulation_seed, .. + } = &mut audit.envelope_audit[real_indices[1]] + else { + unreachable!() + }; + let mut seed: [u8; 32] = BASE64 + .decode(encapsulation_seed.as_str()) + .unwrap() + .try_into() + .unwrap(); + seed[0] ^= 1; + *encapsulation_seed = BASE64.encode(seed); + }); + assert!( + verify_manifest( + &open_manifest(&wrong_seed, &generation_key).unwrap(), + &policy, + None, + &generation_key + ) + .is_err() + ); + + let mut devices = policy.body.devices.clone(); + devices + .iter_mut() + .find(|device| device.public.device_id == reader.device_id().unwrap()) + .unwrap() + .revoked_at = Some(2); + let (revoked, _) = PolicyState::successor(&policy, devices, &owner).unwrap(); + let revoked_entry = rewrite_signed_manifest(&valid, &owner, &generation_key, |_, audit| { + audit.policy_id = revoked.id.clone(); + audit.policy_generation = revoked.body.generation; + }); + assert!( + verify_manifest( + &open_manifest(&revoked_entry, &generation_key).unwrap(), + &revoked, + None, + &generation_key + ) + .is_err() ); - let malicious = serde_json::to_vec(&outer).unwrap(); - let verified = read_manifest_header(&malicious, &policy, None).unwrap(); - assert!(unwrap_generation_key(&verified, &owner).is_err()); } } diff --git a/src/policy.rs b/src/policy.rs index eef6223..5d51e02 100644 --- a/src/policy.rs +++ b/src/policy.rs @@ -10,8 +10,8 @@ use crate::crypto::{ verify_domain, }; -pub const POLICY_FORMAT_VERSION: u32 = 3; -const POLICY_SIGNATURE_DOMAIN: &[u8] = b"git-remote-e2ee policy v4\0"; +pub const POLICY_FORMAT_VERSION: u32 = 5; +const POLICY_SIGNATURE_DOMAIN: &[u8] = b"git-remote-e2ee policy v5\0"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -22,6 +22,14 @@ pub struct DeviceRoles { } impl DeviceRoles { + pub fn reader() -> Self { + Self { + reader: true, + writer: false, + administrator: false, + } + } + pub fn owner() -> Self { Self { reader: true, @@ -265,6 +273,11 @@ impl PolicyState { .is_some_and(|device| device.active() && device.roles.writer) } + pub fn is_active_reader(&self, device_id: &str) -> bool { + self.device(device_id) + .is_some_and(|device| device.active() && device.roles.reader) + } + pub fn is_active_admin(&self, device_id: &str) -> bool { self.device(device_id) .is_some_and(|device| device.active() && device.roles.administrator) @@ -342,6 +355,33 @@ mod tests { assert!(unauthorized.validate_successor(&parent).is_err()); } + #[test] + fn writers_and_administrators_must_also_be_readers() { + let owner = KeyFile::generate(); + let (parent, _) = PolicyState::genesis(&owner).unwrap(); + for roles in [ + DeviceRoles { + reader: false, + writer: true, + administrator: false, + }, + DeviceRoles { + reader: false, + writer: false, + administrator: true, + }, + ] { + let added = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let mut devices = parent.body.devices.clone(); + devices.push(DeviceRecord { + public: added.public_device().unwrap(), + roles, + revoked_at: None, + }); + assert!(PolicyState::successor(&parent, devices, &owner).is_err()); + } + } + #[test] fn genesis_cannot_be_substituted_under_a_known_repository_root() { let expected = KeyFile::generate(); diff --git a/src/repository.rs b/src/repository.rs index b8c20e4..8b1de5f 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -1,12 +1,14 @@ -use std::collections::{BTreeMap, HashSet}; -use std::fs::{self, OpenOptions}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fs::{self, File, OpenOptions}; use std::io::{self, Write}; use std::path::Path; use anyhow::{Context, Result, bail}; +use fs2::FileExt; use rand::RngCore; use rand::rngs::OsRng; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use crate::crypto::{ KeyFile, PublicDevice, SecretKey, SubkeyKind, derive_subkey, object_id, open_pack_stream, @@ -15,26 +17,96 @@ use crate::crypto::{ use crate::git; use crate::manifest::{ Manifest, ManifestAuthorization, ManifestHeader, PackDescriptor, open_manifest, pack_aad, - peek_manifest_header, read_manifest_header, seal_manifest, unwrap_generation_key, - unwrap_predecessor_key, wrap_predecessor_key, + peek_manifest_header, seal_manifest, unwrap_generation_key, unwrap_predecessor_key, + verify_manifest, wrap_predecessor_key, }; use crate::policy::{DeviceRecord, DeviceRoles, PolicyState}; -use crate::storage::{ObjectKind, Storage}; +use crate::storage::{HeadObservation, ObjectKind, Storage}; +use crate::trace; -#[derive(Debug, Default, Serialize, Deserialize)] +const MAX_RECOVERY_OUTER_COMMITS: usize = 2048; +const MAX_RECOVERY_CANDIDATES: usize = 256; +const MAX_RECOVERY_BYTES: u64 = 64 * 1024 * 1024; +const MAX_RECOVERY_MANIFEST_BYTES: u64 = 64 * 1024 * 1024; +const MAX_RECOVERY_CRYPTO_OPERATIONS: u64 = 1_000_000; +const MAX_REJECTED_HEADS: usize = 16; +const STALE_FLOOR_WARNING: &str = "Updates after this floor may be omitted. This may also omit revocations and disclose subsequently published content to devices excluded by a newer policy."; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecoveryClass { + Valid, + Invalid, + Unverifiable, + Unsupported, + Unavailable, + Discontinuous, +} + +impl RecoveryClass { + pub fn name(self) -> &'static str { + match self { + Self::Valid => "valid", + Self::Invalid => "invalid", + Self::Unverifiable => "unverifiable", + Self::Unsupported => "unsupported", + Self::Unavailable => "unavailable", + Self::Discontinuous => "discontinuous", + } + } +} + +#[derive(Clone, Debug)] +pub struct RecoveryCandidate { + pub manifest_id: String, + pub generation: u64, + pub outer_commit: Option, + pub verified_signer: String, +} + +#[derive(Clone, Debug)] +pub struct RecoveryReport { + pub classification: RecoveryClass, + pub reason: String, + pub head_id: Option, + pub claimed_signer: Option, + pub floor_id: Option, + pub floor_generation: Option, + pub candidates: Vec, + pub default_base: Option, + pub replays: Vec, + pub conflict: bool, + pub freshness_unverified: bool, + pub warning: Option, + pub blocked_reason: Option, + pub published_manifest: Option, +} + +pub struct RecoveryOptions<'a> { + pub publish: bool, + pub base: Option<&'a str>, + pub discard_newer: bool, + pub accept_stale_floor: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] struct ClientState { #[serde(default)] format_version: u64, #[serde(default)] packs: Vec, + // Continuity floor: imported and connectivity-verified, or our own CAS winner. head_id: Option, generation: Option, + // The last generation whose pack deltas and advertised refs we imported and checked. #[serde(default)] imported_generation: Option, #[serde(default)] policy_generation: Option, #[serde(default)] repository_root: Option, + #[serde(default)] + verified_refs: BTreeMap, } #[derive(Clone)] @@ -44,6 +116,62 @@ struct OpenedState { header: ManifestHeader, policy: PolicyState, generation_key: SecretKey, + verified_signer: String, +} + +struct CommitHead<'a> { + previous: Option<&'a OpenedState>, + observation: Option<&'a HeadObservation>, +} + +struct ClientStateLock { + _file: File, +} + +struct RecoveryBudget { + crypto_operations: u64, + bytes_read: u64, +} + +impl RecoveryBudget { + fn charge_manifest(&mut self, header: &ManifestHeader, byte_count: usize) -> Result<()> { + self.bytes_read = self + .bytes_read + .checked_add(byte_count as u64) + .context("recovery byte budget overflow")?; + if self.bytes_read > MAX_RECOVERY_BYTES { + bail!("recovery manifest byte budget exhausted") + } + let operations = (header.generation_key_envelopes.len() as u64) + .checked_mul(2) + .and_then(|value| value.checked_add(1)) + .context("recovery crypto budget overflow")?; + self.crypto_operations = self + .crypto_operations + .checked_add(operations) + .context("recovery crypto budget overflow")?; + if self.crypto_operations > MAX_RECOVERY_CRYPTO_OPERATIONS { + bail!("recovery cryptographic-operation budget exhausted") + } + Ok(()) + } +} + +impl ClientStateLock { + fn acquire(path: &Path) -> Result { + // Lock a stable sibling: persist_client_state atomically replaces the state file. + let lock_path = path.with_extension("json.lock"); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .with_context(|| format!("open client state lock {}", lock_path.display()))?; + FileExt::lock_exclusive(&file) + .with_context(|| format!("lock client state {}", path.display()))?; + Ok(Self { _file: file }) + } } pub struct EncryptedRepository { @@ -61,10 +189,8 @@ impl EncryptedRepository { bail!("encrypted repository is already initialized") } let (policy, policy_bytes) = PolicyState::genesis(&self.key)?; - self.storage - .put_object_if_absent(ObjectKind::Policy, &policy.id, &policy_bytes)?; let generation_key = random_key(); - let genesis = Manifest::genesis(self.key.repository_root.clone(), &policy); + let genesis = Manifest::genesis(self.key.repository_root.clone(), &policy, policy_bytes); self.commit_manifest( None, &policy, @@ -81,8 +207,10 @@ impl EncryptedRepository { roles: DeviceRoles, pin_path: &Path, ) -> Result<(String, String)> { - let current = self.current_state()?; - self.validate_pinned_head(¤t, &read_client_state(pin_path)?)?; + let _pin_lock = ClientStateLock::acquire(pin_path)?; + let chain = self.current_chain()?; + let current = chain.first().context("manifest chain is empty")?.clone(); + self.validate_pinned_head(&chain, &read_client_state(pin_path)?)?; self.require_admin(¤t.policy)?; if current.policy.device(&public.device_id).is_some() { bail!("device already exists in policy") @@ -96,21 +224,24 @@ impl EncryptedRepository { let (next_policy, policy_bytes) = PolicyState::successor(¤t.policy, devices, &self.key)?; let next_key = random_key(); - let next = transition_manifest(¤t, &next_policy, &next_key)?; - let result = self.publish_policy_transition( - ¤t, - &next_policy, - &policy_bytes, - &next_key, - next.clone(), + let next = transition_manifest(¤t, &next_policy, policy_bytes, &next_key)?; + let result = + self.publish_policy_transition(¤t, &next_policy, &next_key, next.clone())?; + write_client_state_locked( + pin_path, + &HashSet::new(), + &result.0, + &next, + &BTreeMap::new(), )?; - write_client_state(pin_path, &HashSet::new(), &result.0, &next)?; Ok(result) } pub fn revoke_device(&self, device_id: &str, pin_path: &Path) -> Result<(String, String)> { - let current = self.current_state()?; - self.validate_pinned_head(¤t, &read_client_state(pin_path)?)?; + let _pin_lock = ClientStateLock::acquire(pin_path)?; + let chain = self.current_chain()?; + let current = chain.first().context("manifest chain is empty")?.clone(); + self.validate_pinned_head(&chain, &read_client_state(pin_path)?)?; self.require_admin(¤t.policy)?; let mut devices = current.policy.body.devices.clone(); let target = devices @@ -133,39 +264,53 @@ impl EncryptedRepository { let (next_policy, policy_bytes) = PolicyState::successor(¤t.policy, devices, &self.key)?; let next_key = random_key(); - let next = transition_manifest(¤t, &next_policy, &next_key)?; - let result = self.publish_policy_transition( - ¤t, - &next_policy, - &policy_bytes, - &next_key, - next.clone(), + let next = transition_manifest(¤t, &next_policy, policy_bytes, &next_key)?; + let result = + self.publish_policy_transition(¤t, &next_policy, &next_key, next.clone())?; + write_client_state_locked( + pin_path, + &HashSet::new(), + &result.0, + &next, + &BTreeMap::new(), )?; - write_client_state(pin_path, &HashSet::new(), &result.0, &next)?; Ok(result) } pub fn pin_admin_state(&self, pin_path: &Path) -> Result<()> { - let current = self.current_state()?; + let _pin_lock = ClientStateLock::acquire(pin_path)?; + let chain = self.current_chain()?; + let current = chain.first().context("manifest chain is empty")?; let previous = read_client_state(pin_path)?; - self.validate_pinned_head(¤t, &previous)?; - write_client_state(pin_path, &HashSet::new(), ¤t.id, ¤t.manifest) + self.validate_pinned_head(&chain, &previous)?; + write_client_state_locked( + pin_path, + &HashSet::new(), + ¤t.id, + ¤t.manifest, + &BTreeMap::new(), + ) } pub fn list_devices(&self) -> Result> { - Ok(self.current_state()?.policy.body.devices) + Ok(self + .current_chain()? + .into_iter() + .next() + .context("manifest chain is empty")? + .policy + .body + .devices) } fn publish_policy_transition( &self, current: &OpenedState, policy: &PolicyState, - policy_bytes: &[u8], generation_key: &[u8; 32], manifest: Manifest, ) -> Result<(String, String)> { - self.storage - .put_object_if_absent(ObjectKind::Policy, &policy.id, policy_bytes)?; + policy.validate_successor(¤t.policy)?; let manifest_id = self.commit_manifest( Some(current), policy, @@ -209,6 +354,19 @@ impl EncryptedRepository { self.push_update_inner(repo, Some(remote_name), source_ref, destination_ref, force) } + pub fn delete_ref(&self, repo: &Path, destination_ref: &str) -> Result { + self.change_ref_inner(repo, None, None, destination_ref, false) + } + + pub fn delete_ref_for_remote( + &self, + repo: &Path, + remote_name: &str, + destination_ref: &str, + ) -> Result { + self.change_ref_inner(repo, Some(remote_name), None, destination_ref, false) + } + fn push_update_inner( &self, repo: &Path, @@ -217,50 +375,90 @@ impl EncryptedRepository { destination_ref: &str, force: bool, ) -> Result { - let (current, next_object, non_fast_forward) = - self.preflight_push(repo, remote_name, source_ref, destination_ref, force)?; - if current.manifest.refs.get(destination_ref) == Some(&next_object) { + self.change_ref_inner(repo, remote_name, Some(source_ref), destination_ref, force) + } + + fn change_ref_inner( + &self, + repo: &Path, + remote_name: Option<&str>, + source_ref: Option<&str>, + destination_ref: &str, + force: bool, + ) -> Result { + let state_path = remote_name + .map(|name| client_state_path(repo, name)) + .transpose()?; + let _state_lock = state_path + .as_deref() + .map(ClientStateLock::acquire) + .transpose()?; + let (current, next_object, non_fast_forward) = self.preflight_ref_change( + repo, + remote_name, + state_path.as_deref(), + source_ref, + destination_ref, + force, + )?; + if next_object + .as_ref() + .is_some_and(|next| current.manifest.refs.get(destination_ref) == Some(next)) + { return Ok(current.id); } let mut next_refs = current.manifest.refs.clone(); - next_refs.insert(destination_ref.to_owned(), next_object.clone()); - let mut exclusions = BTreeMap::new(); - if !non_fast_forward { - for (reference, object) in ¤t.manifest.refs { - if git::object_exists(repo, object)? { - exclusions.insert(reference.clone(), object.clone()); - } - } + if let Some(next_object) = &next_object { + next_refs.insert(destination_ref.to_owned(), next_object.clone()); + } else { + next_refs.remove(destination_ref); } - let pack_refs = BTreeMap::from([(destination_ref.to_owned(), next_object)]); let next_key = random_key(); let generation = current.manifest.generation + 1; - let ordinal = 0; - let pack_key = derive_subkey( - &next_key, - ¤t.manifest.repository_root, - generation, - SubkeyKind::Pack, - ordinal, - )?; - let mut pack_source = git::start_incremental_pack(repo, &pack_refs, &exclusions)?; - let mut pack_stage = self.storage.begin_object(ObjectKind::Pack)?; - let sealed = seal_pack_stream( - &pack_key, - &mut pack_source, - &mut pack_stage, - &pack_aad(¤t.manifest.repository_root, generation, ordinal), - )?; - pack_source.finish()?; - pack_stage.finish(&sealed.object_id)?; - let descriptor = PackDescriptor { - id: sealed.object_id, - plaintext_size: sealed.plaintext_size, - generation, - ordinal, - }; + let mut new_packs = Vec::new(); + let mut total_pack_count = current.manifest.total_pack_count; + if let Some(next_object) = next_object { + let mut exclusions = BTreeMap::new(); + if !non_fast_forward { + for (reference, object) in ¤t.manifest.refs { + if git::object_exists(repo, object)? { + exclusions.insert(reference.clone(), object.clone()); + } + } + } + let pack_refs = BTreeMap::from([(destination_ref.to_owned(), next_object)]); + let ordinal = 0; + let pack_key = derive_subkey( + &next_key, + ¤t.manifest.repository_root, + generation, + SubkeyKind::Pack, + ordinal, + )?; + let pack_timer = trace::Span::new("inner_pack_encrypt_and_store"); + let mut pack_source = git::start_incremental_pack(repo, &pack_refs, &exclusions)?; + let mut pack_stage = self.storage.begin_object(ObjectKind::Pack)?; + let sealed = seal_pack_stream( + &pack_key, + &mut pack_source, + &mut pack_stage, + &pack_aad(¤t.manifest.repository_root, generation, ordinal), + )?; + pack_source.finish()?; + let stage_finish_timer = trace::Span::new("inner_pack_stage_finish"); + pack_stage.finish(&sealed.object_id)?; + drop(stage_finish_timer); + drop(pack_timer); + new_packs.push(PackDescriptor { + id: sealed.object_id, + plaintext_size: sealed.plaintext_size, + generation, + ordinal, + }); + total_pack_count += 1; + } let predecessor_key_wrap = wrap_predecessor_key( &next_key, ¤t.generation_key, @@ -276,10 +474,11 @@ impl EncryptedRepository { policy_id: current.policy.id.clone(), policy_generation: current.policy.body.generation, authorization: ManifestAuthorization::Writer, - total_pack_count: current.manifest.total_pack_count + 1, + total_pack_count, refs: next_refs, - new_packs: vec![descriptor], + new_packs, predecessor_key_wrap: Some(predecessor_key_wrap), + introduced_policy: None, }; let next_id = self.commit_manifest( Some(¤t), @@ -289,8 +488,13 @@ impl EncryptedRepository { ManifestAuthorization::Writer, next.clone(), )?; - if let Some(remote_name) = remote_name { - self.pin_manifest(repo, remote_name, &next_id, &next)?; + if let Some(state_path) = state_path.as_deref() { + self.pin_manifest_locked( + state_path, + &next_id, + &next, + Some((repo, ¤t.manifest.refs)), + )?; } Ok(next_id) } @@ -302,7 +506,7 @@ impl EncryptedRepository { destination_ref: &str, force: bool, ) -> Result<()> { - self.preflight_push(repo, None, source_ref, destination_ref, force)?; + self.preflight_ref_change(repo, None, None, Some(source_ref), destination_ref, force)?; Ok(()) } @@ -314,38 +518,95 @@ impl EncryptedRepository { destination_ref: &str, force: bool, ) -> Result<()> { - self.preflight_push(repo, Some(remote_name), source_ref, destination_ref, force)?; + let state_path = client_state_path(repo, remote_name)?; + let _state_lock = ClientStateLock::acquire(&state_path)?; + self.preflight_ref_change( + repo, + Some(remote_name), + Some(&state_path), + Some(source_ref), + destination_ref, + force, + )?; Ok(()) } - fn preflight_push( + pub fn validate_delete_ref_for_remote( + &self, + repo: &Path, + remote_name: &str, + destination_ref: &str, + ) -> Result<()> { + let state_path = client_state_path(repo, remote_name)?; + let _state_lock = ClientStateLock::acquire(&state_path)?; + self.preflight_ref_change( + repo, + Some(remote_name), + Some(&state_path), + None, + destination_ref, + false, + )?; + Ok(()) + } + + fn preflight_ref_change( &self, repo: &Path, remote_name: Option<&str>, - source_ref: &str, + state_path: Option<&Path>, + source_ref: Option<&str>, destination_ref: &str, force: bool, - ) -> Result<(OpenedState, String, bool)> { - if !destination_ref.starts_with("refs/heads/") { - bail!("only refs/heads/* destinations are supported") - } + ) -> Result<(OpenedState, Option, bool)> { git::ensure_repository(repo)?; - let current = self.current_state()?; - if !current.policy.is_active_writer(&self.key.device_id()?) { + let ref_kind = git::validate_inner_ref(repo, destination_ref)?; + let chain = self.current_chain()?; + let current = chain.first().context("manifest chain is empty")?.clone(); + let device_id = self.key.device_id()?; + if !current.policy.is_active_writer(&device_id) { + if current.policy.is_active_reader(&device_id) { + if let Some(remote_name) = remote_name { + bail!( + "this device is read-only for {remote_name}; ask an administrator for the write role" + ) + } + bail!("this device is read-only; ask an administrator for the write role") + } bail!("device is not an active writer") } - if let Some(remote_name) = remote_name { - self.validate_remote_continuity(repo, remote_name, ¤t)?; - } else { - // Direct callers do not have a clone-local pin, so validate the - // complete signed and encrypted history before extending it. - self.walk_chain(current.clone(), None)?; + if remote_name.is_some() { + let state_path = state_path.context("remote client state path is missing")?; + self.validate_remote_continuity(state_path, &chain)?; + } + for reference in current.manifest.refs.keys() { + git::validate_inner_ref(repo, reference)?; } + let old_object = current.manifest.refs.get(destination_ref); + let Some(source_ref) = source_ref else { + if old_object.is_none() { + bail!("remote ref {destination_ref} does not exist") + } + if destination_ref == "refs/heads/main" { + bail!( + "cannot delete refs/heads/main because it is advertised as the remote default branch" + ) + } + return Ok((current, None, false)); + }; + let next_object = git::resolve_ref(repo, source_ref)?; - if current.manifest.refs.get(destination_ref) == Some(&next_object) { - return Ok((current, next_object, false)); + if old_object == Some(&next_object) { + return Ok((current, Some(next_object), false)); } - let non_fast_forward = if let Some(old_object) = current.manifest.refs.get(destination_ref) + if ref_kind == git::InnerRefKind::Branch { + git::ensure_branch_target(repo, destination_ref, &next_object)?; + } + if ref_kind == git::InnerRefKind::Tag && old_object.is_some() && !force { + bail!("tag {destination_ref} already exists; use --force to move it") + } + let non_fast_forward = if ref_kind == git::InnerRefKind::Branch + && let Some(old_object) = old_object { if !git::object_exists(repo, old_object)? { bail!("remote tip for {destination_ref} is missing locally; fetch first") @@ -357,16 +618,23 @@ impl EncryptedRepository { if non_fast_forward && !force { bail!("non-fast-forward update rejected for {destination_ref}") } - Ok((current, next_object, non_fast_forward)) + Ok((current, Some(next_object), non_fast_forward)) } pub fn fetch_into(&self, repo: &Path, remote_name: &str) -> Result { - let manifest = self.import_packs(repo, remote_name)?; + git::ensure_repository(repo)?; + validate_remote_name(remote_name)?; + let state_path = client_state_path(repo, remote_name)?; + let _state_lock = ClientStateLock::acquire(&state_path)?; + let manifest = self.import_packs_locked(repo, &state_path)?; for (reference, object) in &manifest.refs { - let short = reference - .strip_prefix("refs/heads/") - .with_context(|| format!("unsupported remote ref {reference}"))?; - git::update_ref(repo, &format!("refs/remotes/{remote_name}/{short}"), object)?; + match git::validate_inner_ref(repo, reference)? { + git::InnerRefKind::Branch => { + let short = reference.strip_prefix("refs/heads/").unwrap(); + git::update_ref(repo, &format!("refs/remotes/{remote_name}/{short}"), object)?; + } + git::InnerRefKind::Tag => git::update_ref(repo, reference, object)?, + } } Ok(manifest) } @@ -374,27 +642,58 @@ impl EncryptedRepository { pub fn observe_manifest(&self, repo: &Path, remote_name: &str) -> Result { git::ensure_repository(repo)?; validate_remote_name(remote_name)?; - let current = self.current_state()?; let state_path = client_state_path(repo, remote_name)?; + let _state_lock = ClientStateLock::acquire(&state_path)?; + let chain = self.current_chain()?; + let current = chain.first().context("manifest chain is empty")?.clone(); + for reference in current.manifest.refs.keys() { + git::validate_inner_ref(repo, reference)?; + } let state = read_client_state(&state_path)?; - self.validate_pinned_head(¤t, &state)?; - write_observed_client_state(&state_path, state, ¤t.id, ¤t.manifest)?; - Ok(current.manifest) + self.validate_pinned_head(&chain, &state)?; + // Git skips the helper's fetch command when every advertised object is + // already local, for example after a membership change, a deletion, or + // a tag that points at an existing object. Advance the floor here only + // when the advertised refs are fully connected locally, which is the + // same check a fetch would finish with; authentication alone never + // moves it. Earlier packs are not needed once these refs are + // connected, because later packs are built against these tips. + if git::ensure_refs_connected_since(repo, ¤t.manifest.refs, &state.verified_refs) + .is_ok() + { + let packs: HashSet = state.packs.iter().cloned().collect(); + write_client_state_locked( + &state_path, + &packs, + ¤t.id, + ¤t.manifest, + ¤t.manifest.refs, + )?; + } + Ok(current.manifest.clone()) } pub fn import_packs(&self, repo: &Path, remote_name: &str) -> Result { git::ensure_repository(repo)?; validate_remote_name(remote_name)?; - let current = self.current_state()?; let state_path = client_state_path(repo, remote_name)?; - let state = read_client_state(&state_path)?; - self.validate_pinned_head(¤t, &state)?; + let _state_lock = ClientStateLock::acquire(&state_path)?; + self.import_packs_locked(repo, &state_path) + } + + fn import_packs_locked(&self, repo: &Path, state_path: &Path) -> Result { + let chain = self.current_chain()?; + let current = chain.first().context("manifest chain is empty")?.clone(); + for reference in current.manifest.refs.keys() { + git::validate_inner_ref(repo, reference)?; + } + let state = read_client_state(state_path)?; + self.validate_pinned_head(&chain, &state)?; let stop_generation = state.imported_generation; - let chain = self.walk_chain(current.clone(), stop_generation)?; let mut imported: HashSet = state.packs.into_iter().collect(); for entry in chain.iter().rev() { - if stop_generation == Some(entry.manifest.generation) { + if stop_generation.is_some_and(|generation| entry.manifest.generation <= generation) { continue; } for descriptor in &entry.manifest.new_packs { @@ -413,6 +712,7 @@ impl EncryptedRepository { descriptor.ordinal, )?; let mut importer = git::start_pack_import(repo)?; + let decrypt_timer = trace::Span::new("pack_decrypt_to_index_pack"); open_pack_stream( &pack_key, encrypted, @@ -425,38 +725,51 @@ impl EncryptedRepository { descriptor.plaintext_size, &descriptor.id, )?; + drop(decrypt_timer); importer.finish()?; imported.insert(descriptor.id.clone()); } } - git::ensure_refs_connected(repo, ¤t.manifest.refs)?; - write_client_state(&state_path, &imported, ¤t.id, ¤t.manifest)?; + let connectivity_timer = trace::Span::new("inner_ref_connectivity_check"); + git::ensure_refs_connected_since(repo, ¤t.manifest.refs, &state.verified_refs)?; + drop(connectivity_timer); + write_client_state_locked( + state_path, + &imported, + ¤t.id, + ¤t.manifest, + ¤t.manifest.refs, + )?; Ok(current.manifest) } - fn validate_remote_continuity( - &self, - repo: &Path, - remote_name: &str, - current: &OpenedState, - ) -> Result<()> { - let state = read_client_state(&client_state_path(repo, remote_name)?)?; - self.validate_pinned_head(current, &state) + fn validate_remote_continuity(&self, state_path: &Path, chain: &[OpenedState]) -> Result<()> { + let state = read_client_state(state_path)?; + self.validate_pinned_head(chain, &state) } - fn pin_manifest( + fn pin_manifest_locked( &self, - repo: &Path, - remote_name: &str, + state_path: &Path, head_id: &str, manifest: &Manifest, + published_from: Option<(&Path, &BTreeMap)>, ) -> Result<()> { - let state_path = client_state_path(repo, remote_name)?; - let state = read_client_state(&state_path)?; - write_observed_client_state(&state_path, state, head_id, manifest) + let mut state = read_client_state(state_path)?; + if let Some((repo, previous_refs)) = published_from + && state.verified_refs == *previous_refs + && git::has_complete_object_graph(repo) + { + // The completed local pack walk covers new reachability only beyond + // refs already verified by this client. Shallow and promisor repos + // can omit history, so they must not advance this frontier. + state.verified_refs = manifest.refs.clone(); + } + write_published_client_state_locked(state_path, state, head_id, manifest) } - fn validate_pinned_head(&self, current: &OpenedState, state: &ClientState) -> Result<()> { + fn validate_pinned_head(&self, chain: &[OpenedState], state: &ClientState) -> Result<()> { + let current = chain.first().context("manifest chain is empty")?; if let Some(root) = &state.repository_root && root != ¤t.manifest.repository_root { @@ -470,7 +783,6 @@ impl EncryptedRepository { let (Some(pinned_id), Some(pinned_generation)) = (state.head_id.as_deref(), state.generation) else { - self.walk_chain(current.clone(), None)?; return Ok(()); }; if current.manifest.generation < pinned_generation { @@ -479,9 +791,9 @@ impl EncryptedRepository { current.manifest.generation ) } - let chain = self.walk_chain(current.clone(), Some(pinned_generation))?; let pinned = chain - .last() + .iter() + .find(|entry| entry.manifest.generation == pinned_generation) .context("remote manifest chain ended before pinned generation")?; if pinned.id != pinned_id { bail!("remote manifest history forked from the locally pinned head") @@ -490,8 +802,8 @@ impl EncryptedRepository { } pub fn verify(&self) -> Result { - let current = self.current_state()?; - let chain = self.walk_chain(current.clone(), None)?; + let chain = self.current_chain()?; + let current = chain.first().context("manifest chain is empty")?; let mut count = 0_u64; for entry in chain.iter().rev() { for pack in &entry.manifest.new_packs { @@ -521,137 +833,715 @@ impl EncryptedRepository { if count != current.manifest.total_pack_count { bail!("manifest delta log pack count mismatch") } - Ok(current.manifest) + Ok(current.manifest.clone()) } pub fn current_manifest(&self) -> Result<(String, Manifest)> { - let current = self.current_state()?; + let current = self + .current_chain()? + .into_iter() + .next() + .context("manifest chain is empty")?; Ok((current.id, current.manifest)) } - fn current_state(&self) -> Result { + pub fn recover( + &self, + repo: &Path, + remote_name: &str, + options: RecoveryOptions<'_>, + ) -> Result { + git::ensure_repository(repo)?; + validate_remote_name(remote_name)?; + let state_path = client_state_path(repo, remote_name)?; + let _state_lock = ClientStateLock::acquire(&state_path)?; + let state = read_client_state(&state_path)?; + let mut report = RecoveryReport { + classification: RecoveryClass::Unavailable, + reason: "could not read the storage head".to_owned(), + head_id: None, + claimed_signer: None, + floor_id: state.head_id.clone(), + floor_generation: state.generation, + candidates: Vec::new(), + default_base: None, + replays: Vec::new(), + conflict: false, + freshness_unverified: state.head_id.is_none() || state.generation.is_none(), + warning: None, + blocked_reason: None, + published_manifest: None, + }; + let observation = match self.storage.observe_head() { + Ok(observation) => observation, + Err(error) => { + if format!("{error:#}").contains("budget exhausted") { + return Err(error).context("bounded recovery discovery failed"); + } + report.reason = format!("storage head is unavailable: {error:#}"); + if options.publish { + report.blocked_reason = Some(report.reason.clone()); + } + return Ok(report); + } + }; + report.head_id = observation.head_id.clone(); + let mut budget = RecoveryBudget { + crypto_operations: 0, + bytes_read: 0, + }; + let current_chain = match observation.head_id.as_deref() { + Some(head) => self.chain_from( + head, + &mut |id| self.read_manifest_material(id), + Some(&mut budget), + ), + None if observation.head_bytes.is_some() => Err(anyhow::anyhow!( + "storage HEAD is not a valid manifest object ID" + )), + None => Err(anyhow::anyhow!("encrypted repository has no storage HEAD")), + }; + match current_chain { + Ok(chain) => { + report.claimed_signer = chain.first().map(|entry| entry.verified_signer.clone()); + if state.head_id.is_some() && state.generation.is_some() { + match self.validate_pinned_head(&chain, &state) { + Ok(()) => match self.check_chain_ciphertext_presence(&chain) { + Ok(()) => { + report.classification = RecoveryClass::Valid; + report.reason = + "storage head is authenticated and continuous with the local floor" + .to_owned(); + } + Err(error) => { + report.classification = RecoveryClass::Unavailable; + report.reason = format!( + "authenticated storage head has unavailable ciphertext: {error:#}" + ); + } + }, + Err(error) => { + report.classification = RecoveryClass::Discontinuous; + report.reason = + format!("authenticated storage head is discontinuous: {error:#}"); + } + } + } else { + match self.check_chain_ciphertext_presence(&chain) { + Ok(()) => { + report.classification = RecoveryClass::Valid; + report.reason = + "storage head is authenticated, but this client has no continuity floor" + .to_owned(); + } + Err(error) => { + report.classification = RecoveryClass::Unavailable; + report.reason = format!( + "authenticated storage head has unavailable ciphertext: {error:#}" + ); + } + } + report.freshness_unverified = true; + } + } + Err(error) => { + if format!("{error:#}").contains("budget exhausted") { + return Err(error).context("bounded recovery discovery failed"); + } + let (class, reason) = + classify_recovery_error(&error, observation.head_bytes.as_deref()); + report.classification = class; + report.reason = reason; + if class == RecoveryClass::Invalid + && let Some(head) = observation.head_id.as_deref() + { + report.claimed_signer = self.claimed_signer_for_head(head).ok(); + } + } + } + + if report.classification == RecoveryClass::Invalid { + let evidence_id = observation.head_id.clone().unwrap_or_else(|| { + hex::encode(Sha256::digest( + observation.head_bytes.as_deref().unwrap_or_default(), + )) + }); + record_rejected_head( + &state_path, + &evidence_id, + report.classification, + &report.reason, + )?; + } else { + report.blocked_reason = Some(match report.classification { + RecoveryClass::Valid => { + "the storage head is already valid; recovery is not needed".to_owned() + } + RecoveryClass::Unverifiable => { + "the storage head cannot be opened by this device; recovery is forbidden" + .to_owned() + } + RecoveryClass::Unsupported => { + "the storage head uses an unsupported format; recovery is forbidden".to_owned() + } + RecoveryClass::Unavailable => { + "required storage data is unavailable; recovery is forbidden".to_owned() + } + RecoveryClass::Discontinuous => { + "the storage head is discontinuous with the local floor; recovery is forbidden" + .to_owned() + } + RecoveryClass::Invalid => unreachable!(), + }); + return Ok(report); + } + + let (Some(floor_id), Some(floor_generation)) = (state.head_id.as_deref(), state.generation) + else { + report.freshness_unverified = true; + report.blocked_reason = Some( + "this repository has no authenticated continuity floor; freshness and fork identity are unverified".to_owned(), + ); + return Ok(report); + }; + + let history = self + .storage + .recovery_history(MAX_RECOVERY_OUTER_COMMITS, MAX_RECOVERY_MANIFEST_BYTES)?; + let carrier_history = history.is_some(); + let history = history.unwrap_or_default(); + if history.commits.len() > MAX_RECOVERY_OUTER_COMMITS { + bail!("recovery outer-commit budget exhausted") + } + let historical_bytes = history.manifests.values().try_fold(0_u64, |total, bytes| { + total + .checked_add(bytes.len() as u64) + .context("recovery manifest byte budget overflow") + })?; + if historical_bytes > MAX_RECOVERY_MANIFEST_BYTES { + bail!("recovery manifest byte budget exhausted") + } + budget.bytes_read = budget + .bytes_read + .checked_add(historical_bytes) + .context("recovery manifest byte budget overflow")?; + if budget.bytes_read > MAX_RECOVERY_BYTES { + bail!("recovery manifest byte budget exhausted") + } + if carrier_history + && !history + .commits + .iter() + .any(|commit| commit.head_id.as_deref() == Some(floor_id)) + { + report.blocked_reason = Some(format!( + "carrier history does not contain the pinned floor manifest {floor_id}; refusing discontinuous recovery" + )); + report.classification = RecoveryClass::Discontinuous; + return Ok(report); + } + + let read_history_material = |id: &str| -> Result<(String, Vec, ManifestHeader)> { + match self.read_manifest_material(id) { + Ok(material) => Ok(material), + Err(current_error) => { + let bytes = history.manifests.get(id).with_context(|| { + format!("historical manifest {id} is unavailable (current read: {current_error:#})") + })?; + if object_id(bytes) != id { + bail!("historical manifest {id} content ID mismatch") + } + let header = peek_manifest_header(bytes)?; + Ok((id.to_owned(), bytes.clone(), header)) + } + } + }; + let floor_chain = match self.chain_from( + floor_id, + &mut |id| read_history_material(id), + Some(&mut budget), + ) { + Ok(chain) => chain, + Err(error) => { + report.blocked_reason = Some(format!( + "pinned floor is unavailable or cannot be authenticated: {error:#}" + )); + return Ok(report); + } + }; + let floor = match floor_chain + .iter() + .find(|entry| entry.id == floor_id && entry.manifest.generation == floor_generation) + { + Some(floor) => floor, + None => { + report.blocked_reason = Some( + "client state floor does not match its authenticated manifest generation" + .to_owned(), + ); + return Ok(report); + } + }; + if state + .repository_root + .as_deref() + .is_some_and(|root| root != floor.manifest.repository_root) + || state + .policy_generation + .is_some_and(|generation| generation > floor.manifest.policy_generation) + { + report.blocked_reason = Some( + "client state floor metadata does not match the authenticated floor".to_owned(), + ); + return Ok(report); + } + let floor_candidate = RecoveryCandidate { + manifest_id: floor.id.clone(), + generation: floor.manifest.generation, + outer_commit: history + .commits + .iter() + .find(|commit| commit.head_id.as_deref() == Some(floor_id)) + .map(|commit| commit.commit_id.clone()), + verified_signer: floor.verified_signer.clone(), + }; + + let mut replay_ids = BTreeSet::new(); + let mut seen_head_ids = HashSet::new(); + let mut descendants: Vec<(RecoveryCandidate, Vec)> = Vec::new(); + if carrier_history { + let mut unique_heads = Vec::new(); + for commit in &history.commits { + if let Some(head) = &commit.head_id { + if seen_head_ids.insert(head.clone()) { + unique_heads.push((head, commit)); + } else { + replay_ids.insert(head.clone()); + } + } + } + if unique_heads.len() > MAX_RECOVERY_CANDIDATES { + bail!("recovery candidate budget exhausted") + } + for (candidate_id, commit) in unique_heads { + if candidate_id == floor_id { + continue; + } + let candidate_chain = match self.chain_from( + candidate_id, + &mut |id| read_history_material(id), + Some(&mut budget), + ) { + Ok(chain) => chain, + Err(error) => { + let (class, _) = classify_recovery_error(&error, None); + match class { + RecoveryClass::Invalid => continue, + RecoveryClass::Unverifiable + | RecoveryClass::Unsupported + | RecoveryClass::Unavailable => { + report.blocked_reason = Some(format!( + "historical carrier head {candidate_id} is {}: {error:#}", + class.name() + )); + return Ok(report); + } + RecoveryClass::Discontinuous | RecoveryClass::Valid => continue, + } + } + }; + let candidate = candidate_chain + .first() + .context("validated candidate chain is empty")?; + let candidate_in_floor = floor_chain.iter().any(|entry| entry.id == *candidate_id); + if candidate.manifest.generation < floor_generation && candidate_in_floor { + replay_ids.insert(candidate_id.clone()); + continue; + } + let floor_position = candidate_chain.iter().position(|entry| { + entry.id == floor_id && entry.manifest.generation == floor_generation + }); + if floor_position.is_none() { + report.blocked_reason = Some(format!( + "authenticated historical head {candidate_id} does not descend from the pinned floor" + )); + return Ok(report); + } + descendants.push(( + RecoveryCandidate { + manifest_id: candidate.id.clone(), + generation: candidate.manifest.generation, + outer_commit: Some(commit.commit_id.clone()), + verified_signer: candidate.verified_signer.clone(), + }, + candidate_chain, + )); + } + for (index, (left, left_chain)) in descendants.iter().enumerate() { + for (right, right_chain) in descendants.iter().skip(index + 1) { + let left_is_ancestor = + right_chain.iter().any(|entry| entry.id == left.manifest_id); + let right_is_ancestor = + left_chain.iter().any(|entry| entry.id == right.manifest_id); + if !left_is_ancestor && !right_is_ancestor { + report.conflict = true; + } + } + } + } + if report.conflict { + report.blocked_reason = Some( + "authenticated descendants conflict; refusing to choose between forks".to_owned(), + ); + report.candidates.push(floor_candidate); + report + .candidates + .extend(descendants.into_iter().map(|(candidate, _)| candidate)); + report.replays = replay_ids.into_iter().collect(); + return Ok(report); + } + + let mut offers = vec![(floor_candidate.clone(), floor_chain.clone())]; + offers.extend(descendants); + offers.sort_by_key(|(candidate, _)| candidate.generation); + report.candidates = offers + .iter() + .map(|(candidate, _)| candidate.clone()) + .collect(); + report.replays = replay_ids.into_iter().collect(); + let default_offer = offers.last().context("recovery floor offer is missing")?; + report.default_base = Some(default_offer.0.manifest_id.clone()); + if !carrier_history { + report.warning = Some(STALE_FLOOR_WARNING.to_owned()); + } + if !options.publish { + return Ok(report); + } + + let selected_id = options.base.unwrap_or(&default_offer.0.manifest_id); + let selected = match offers + .iter() + .find(|(candidate, _)| candidate.manifest_id == selected_id) + { + Some(selected) => selected, + None => { + report.blocked_reason = Some(format!( + "selected base {selected_id} is not an authenticated recovery offer" + )); + return Ok(report); + } + }; + if selected.0.generation < default_offer.0.generation && !options.discard_newer { + report.blocked_reason = Some(format!( + "choosing older base {} requires --discard-newer", + selected.0.manifest_id + )); + return Ok(report); + } + if !carrier_history && !options.accept_stale_floor { + report.blocked_reason = Some(format!( + "directory recovery requires --accept-stale-floor. {}", + STALE_FLOOR_WARNING + )); + return Ok(report); + } + let base_state = selected + .1 + .first() + .filter(|entry| entry.id == selected.0.manifest_id) + .context("selected recovery base is missing from its validated chain")?; + if !base_state.policy.is_active_writer(&self.key.device_id()?) { + report.blocked_reason = + Some("device is not an active writer under the selected base policy".to_owned()); + return Ok(report); + } + if let Err(error) = self.restore_base_objects(&selected.1) { + report.blocked_reason = Some(format!("required ciphertext is unavailable: {error:#}")); + return Ok(report); + } + if let Err(error) = self.verify_base_graph(&selected.1) { + report.blocked_reason = Some(format!( + "selected base does not resolve to a complete local object graph: {error:#}" + )); + return Ok(report); + } + let required_manifests: Vec<_> = selected.1.iter().map(|entry| entry.id.clone()).collect(); + let required_packs: Vec<_> = selected + .1 + .iter() + .flat_map(|entry| entry.manifest.new_packs.iter().map(|pack| pack.id.clone())) + .collect(); + self.storage + .prepare_recovery(&required_manifests, &required_packs)?; + + let next_key = random_key(); + let generation = base_state.manifest.generation + 1; + let next = Manifest { + format_version: base_state.manifest.format_version, + repository_root: base_state.manifest.repository_root.clone(), + generation, + previous: Some(base_state.id.clone()), + policy_id: base_state.policy.id.clone(), + policy_generation: base_state.policy.body.generation, + authorization: ManifestAuthorization::Writer, + total_pack_count: base_state.manifest.total_pack_count, + refs: base_state.manifest.refs.clone(), + new_packs: Vec::new(), + predecessor_key_wrap: Some(wrap_predecessor_key( + &next_key, + &base_state.generation_key, + &base_state.manifest.repository_root, + generation, + &base_state.id, + )?), + introduced_policy: None, + }; + let recovered_id = self.commit_manifest_with_observation( + CommitHead { + previous: Some(base_state), + observation: Some(&observation), + }, + &base_state.policy, + None, + &next_key, + ManifestAuthorization::Writer, + next.clone(), + )?; + self.pin_manifest_locked(&state_path, &recovered_id, &next, None)?; + report.published_manifest = Some(recovered_id); + report.blocked_reason = None; + Ok(report) + } + + fn claimed_signer_for_head(&self, head: &str) -> Result { + let (_, encrypted, header) = self.read_manifest_material(head)?; + let key = unwrap_generation_key(&header, &self.key)?; + Ok(open_manifest(&encrypted, &key)?.claimed_signer_id) + } + + fn check_chain_ciphertext_presence(&self, chain: &[OpenedState]) -> Result<()> { + for state in chain { + for descriptor in &state.manifest.new_packs { + self.storage + .open_object(ObjectKind::Pack, &descriptor.id) + .with_context(|| format!("required pack {} is unavailable", descriptor.id))?; + } + } + Ok(()) + } + + fn restore_base_objects(&self, chain: &[OpenedState]) -> Result<()> { + for state in chain { + self.ensure_object_available(ObjectKind::Manifest, &state.id)?; + for descriptor in &state.manifest.new_packs { + self.ensure_object_available(ObjectKind::Pack, &descriptor.id)?; + } + } + Ok(()) + } + + fn verify_base_graph(&self, chain: &[OpenedState]) -> Result<()> { + let temporary = tempfile::tempdir()?; + let init = std::process::Command::new("git") + .arg("init") + .arg("--bare") + .arg("--quiet") + .arg(temporary.path()) + .output()?; + if !init.status.success() { + bail!( + "git init for recovery connectivity check failed: {}", + String::from_utf8_lossy(&init.stderr).trim() + ) + } + for entry in chain.iter().rev() { + for descriptor in &entry.manifest.new_packs { + let encrypted = self.storage.open_object(ObjectKind::Pack, &descriptor.id)?; + let pack_key = derive_subkey( + &entry.generation_key, + &entry.manifest.repository_root, + descriptor.generation, + SubkeyKind::Pack, + descriptor.ordinal, + )?; + let mut importer = git::start_pack_import(temporary.path())?; + open_pack_stream( + &pack_key, + encrypted, + &mut importer, + &pack_aad( + &entry.manifest.repository_root, + descriptor.generation, + descriptor.ordinal, + ), + descriptor.plaintext_size, + &descriptor.id, + )?; + importer.finish()?; + } + } + git::ensure_refs_connected_since( + temporary.path(), + &chain + .first() + .context("selected recovery chain is empty")? + .manifest + .refs, + &BTreeMap::new(), + ) + } + + fn ensure_object_available(&self, kind: ObjectKind, id: &str) -> Result<()> { + if self.storage_object_matches(kind, id)? { + return Ok(()); + } + if self.storage.restore_historical_object(kind, id)? + && self.storage_object_matches(kind, id)? + { + return Ok(()); + } + bail!("required {:?} object {id} is missing or corrupt", kind) + } + + fn storage_object_matches(&self, kind: ObjectKind, id: &str) -> Result { + let mut reader = match self.storage.open_object(kind, id) { + Ok(reader) => reader, + Err(_) => return Ok(false), + }; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(hex::encode(hasher.finalize()) == id) + } + + fn current_chain(&self) -> Result> { + let _timer = trace::Span::new("manifest_chain_authentication"); let head = self .storage .read_head()? .context("encrypted repository is not initialized")?; - let (encrypted, header, policy, parent_policy) = self.read_manifest_material(&head)?; - let generation_key = unwrap_generation_key(&header, &self.key)?; - let manifest = open_manifest(&encrypted, &policy, parent_policy.as_ref(), &generation_key)?; - if manifest.generation == 0 { - manifest.validate_genesis(&policy)?; - } - Ok(OpenedState { - id: head, - manifest, - header, - policy, - generation_key, - }) + self.chain_from(&head, &mut |id| self.read_manifest_material(id), None) } - fn walk_chain( + fn chain_from( &self, - mut current: OpenedState, - stop_generation: Option, - ) -> Result> { - let mut chain = Vec::new(); + head: &str, + read_material: &mut F, + mut budget: Option<&mut RecoveryBudget>, + ) -> Result> + where + F: FnMut(&str) -> Result<(String, Vec, ManifestHeader)>, + { + let (mut id, mut encrypted, mut header) = read_material(head)?; + if let Some(budget) = budget.as_deref_mut() { + budget.charge_manifest(&header, encrypted.len())?; + } + let mut generation_key = unwrap_generation_key(&header, &self.key)?; + let mut opened = open_manifest(&encrypted, &generation_key)?; + let mut unverified = Vec::new(); let mut seen = HashSet::new(); - let mut seen_pack_ids = HashSet::new(); + loop { - if !seen.insert(current.id.clone()) { + if !seen.insert(id.clone()) { bail!("manifest chain contains a cycle") } - if stop_generation.is_some_and(|generation| current.manifest.generation < generation) { - bail!("manifest chain skipped below requested generation") - } - for pack in ¤t.manifest.new_packs { - if !seen_pack_ids.insert(pack.id.clone()) { - bail!("manifest chain repeats pack {}", pack.id) + let Some(previous_id) = opened.manifest.previous.clone() else { + if opened.manifest.generation != 0 { + bail!("manifest chain terminated above generation zero") } - } - let should_stop = stop_generation == Some(current.manifest.generation); - let is_genesis = current.manifest.generation == 0; - chain.push(current.clone()); - if should_stop { + unverified.push((id, opened, generation_key)); break; + }; + let (parent_id, parent_encrypted, parent_header) = read_material(&previous_id)?; + if let Some(budget) = budget.as_deref_mut() { + budget.charge_manifest(&parent_header, parent_encrypted.len())?; } - if is_genesis { - if stop_generation.is_some() { - bail!("manifest chain ended before requested generation") - } - current.manifest.validate_genesis(¤t.policy)?; - break; + let parent_key = + unwrap_predecessor_key(&generation_key, &opened.manifest, &parent_header)?; + unverified.push((id, opened, generation_key)); + id = parent_id; + encrypted = parent_encrypted; + header = parent_header; + generation_key = parent_key; + opened = open_manifest(&encrypted, &generation_key)?; + if opened.header != header { + bail!("predecessor manifest header changed while opening") } + } - let previous_id = current - .manifest - .previous - .clone() - .context("manifest chain terminated above generation zero")?; - let (encrypted, previous_header, previous_policy, previous_parent_policy) = - self.read_manifest_material(&previous_id)?; - let previous_key = unwrap_predecessor_key( - ¤t.generation_key, - ¤t.manifest, - &previous_header, - )?; - let previous_manifest = open_manifest( - &encrypted, - &previous_policy, - previous_parent_policy.as_ref(), - &previous_key, - )?; - current.manifest.validate_successor( - &previous_id, - &previous_manifest, - ¤t.policy, - )?; - current = OpenedState { - id: previous_id, - manifest: previous_manifest, - header: previous_header, - policy: previous_policy, - generation_key: previous_key, + let mut chain_oldest_first: Vec = Vec::with_capacity(unverified.len()); + let mut seen_pack_ids = HashSet::new(); + for (index, (id, opened, generation_key)) in unverified.into_iter().rev().enumerate() { + let manifest = &opened.manifest; + for pack in &manifest.new_packs { + if !seen_pack_ids.insert(pack.id.clone()) { + bail!("manifest chain repeats pack {}", pack.id) + } + } + let (policy, parent_policy) = if index == 0 { + let policy_bytes = manifest + .introduced_policy + .as_deref() + .context("genesis manifest is missing its introduced policy")?; + let policy = PolicyState::parse(policy_bytes)?; + policy.validate_genesis(&self.key.repository_root)?; + manifest.validate_genesis(&policy)?; + (policy, None) + } else { + let parent = chain_oldest_first + .last() + .context("manifest parent is missing")?; + let (policy, parent_policy) = match manifest.authorization { + ManifestAuthorization::Writer => { + if manifest.introduced_policy.is_some() { + bail!("ordinary manifest unexpectedly introduces a policy") + } + (parent.policy.clone(), None) + } + ManifestAuthorization::PolicyTransition => { + let bytes = manifest + .introduced_policy + .as_deref() + .context("policy transition is missing its introduced policy")?; + let policy = PolicyState::parse(bytes)?; + policy.validate_successor(&parent.policy)?; + (policy, Some(parent.policy.clone())) + } + ManifestAuthorization::Checkpoint => { + bail!("checkpoint transitions are reserved but not implemented") + } + }; + manifest.validate_successor(&parent.id, &parent.manifest, &policy)?; + (policy, parent_policy) }; + verify_manifest(&opened, &policy, parent_policy.as_ref(), &generation_key)?; + let verified_signer = opened.claimed_signer_id.clone(); + chain_oldest_first.push(OpenedState { + id, + manifest: opened.manifest, + header: opened.header, + policy, + generation_key, + verified_signer, + }); } - Ok(chain) + chain_oldest_first.reverse(); + Ok(chain_oldest_first) } - fn read_manifest_material( - &self, - id: &str, - ) -> Result<(Vec, ManifestHeader, PolicyState, Option)> { + fn read_manifest_material(&self, id: &str) -> Result<(String, Vec, ManifestHeader)> { let bytes = self.storage.get_object(ObjectKind::Manifest, id)?; if object_id(&bytes) != id { bail!("manifest ciphertext hash mismatch") } - let unverified = peek_manifest_header(&bytes)?; - let policy = self.read_policy(&unverified.policy_id)?; - let parent = match &policy.body.previous { - Some(parent_id) => Some(self.read_policy(parent_id)?), - None => None, - }; - let header = read_manifest_header(&bytes, &policy, parent.as_ref())?; - Ok((bytes, header, policy, parent)) - } - - fn read_policy(&self, id: &str) -> Result { - let bytes = self.storage.get_object(ObjectKind::Policy, id)?; - if object_id(&bytes) != id { - bail!("policy object hash mismatch") - } - let policy = PolicyState::parse(&bytes)?; - if policy.id != id { - bail!("policy id mismatch") - } - match &policy.body.previous { - Some(previous_id) => { - let parent = self.read_policy(previous_id)?; - policy.validate_successor(&parent)?; - } - None => policy.validate_genesis(&self.key.repository_root)?, - } - Ok(policy) + let header = peek_manifest_header(&bytes)?; + Ok((id.to_owned(), bytes, header)) } fn commit_manifest( @@ -663,7 +1553,30 @@ impl EncryptedRepository { authorization: ManifestAuthorization, manifest: Manifest, ) -> Result { - match previous { + self.commit_manifest_with_observation( + CommitHead { + previous, + observation: None, + }, + policy, + parent_policy, + generation_key, + authorization, + manifest, + ) + } + + fn commit_manifest_with_observation( + &self, + head: CommitHead<'_>, + policy: &PolicyState, + parent_policy: Option<&PolicyState>, + generation_key: &[u8; 32], + authorization: ManifestAuthorization, + manifest: Manifest, + ) -> Result { + let manifest_timer = trace::Span::new("manifest_seal_and_self_verify"); + match head.previous { Some(previous) => { manifest.validate_successor(&previous.id, &previous.manifest, policy)? } @@ -680,19 +1593,28 @@ impl EncryptedRepository { self.storage .put_object_if_absent(ObjectKind::Manifest, &id, &encrypted)?; - let header = read_manifest_header(&encrypted, policy, parent_policy)?; - let opened = open_manifest(&encrypted, policy, parent_policy, generation_key)?; - if let Some(previous) = previous { - let recovered = unwrap_predecessor_key(generation_key, &opened, &previous.header)?; + let opened = open_manifest(&encrypted, generation_key)?; + verify_manifest(&opened, policy, parent_policy, generation_key)?; + if let Some(previous) = head.previous { + let recovered = + unwrap_predecessor_key(generation_key, &opened.manifest, &previous.header)?; if *recovered != *previous.generation_key { bail!("new manifest predecessor link did not recover the parent key") } } - if header.generation != manifest.generation { + if opened.header.generation != manifest.generation { bail!("new manifest self-check changed generation") } - self.storage - .compare_and_swap_head(previous.map(|state| state.id.as_str()), &id)?; + drop(manifest_timer); + let cas_timer = trace::Span::new("manifest_compare_and_swap_publish"); + if let Some(observation) = head.observation { + self.storage + .compare_and_swap_observed_head(observation, &id)?; + } else { + self.storage + .compare_and_swap_head(head.previous.map(|state| state.id.as_str()), &id)?; + } + drop(cas_timer); Ok(id) } } @@ -700,6 +1622,7 @@ impl EncryptedRepository { fn transition_manifest( current: &OpenedState, policy: &PolicyState, + policy_bytes: Vec, generation_key: &[u8; 32], ) -> Result { let generation = current.manifest.generation + 1; @@ -721,9 +1644,93 @@ fn transition_manifest( generation, ¤t.id, )?), + introduced_policy: Some(policy_bytes), }) } +#[derive(Clone, Debug, Serialize, Deserialize)] +struct RejectedHeadEvidence { + head_id: String, + classification: RecoveryClass, + reason: String, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct RejectedHeadLog { + format_version: u32, + entries: Vec, +} + +fn classify_recovery_error( + error: &anyhow::Error, + raw_head: Option<&[u8]>, +) -> (RecoveryClass, String) { + let detail = format!("{error:#}"); + if detail.contains("unsupported") || detail.contains("reserved but not implemented") { + return (RecoveryClass::Unsupported, detail); + } + if detail.contains("device is not an active reader") + || detail.contains("cannot open the generation key") + { + return (RecoveryClass::Unverifiable, detail); + } + if detail.contains("key file belongs to a different repository") { + return (RecoveryClass::Discontinuous, detail); + } + if error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|source| source.kind() != io::ErrorKind::InvalidData) + }) { + return (RecoveryClass::Unavailable, detail); + } + if detail.contains("encrypted repository has no storage HEAD") && raw_head.is_none() { + return (RecoveryClass::Unavailable, detail); + } + (RecoveryClass::Invalid, detail) +} + +fn record_rejected_head( + state_path: &Path, + head_id: &str, + classification: RecoveryClass, + reason: &str, +) -> Result<()> { + let directory = state_path + .parent() + .context("client state path has no parent")?; + let path = directory.join("rejected-heads.json"); + let mut log = match fs::read(&path) { + Ok(bytes) => serde_json::from_slice::(&bytes).unwrap_or_default(), + Err(error) if error.kind() == io::ErrorKind::NotFound => RejectedHeadLog::default(), + Err(error) => return Err(error.into()), + }; + log.format_version = 1; + log.entries.retain(|entry| entry.head_id != head_id); + let reason: String = reason.chars().take(512).collect(); + log.entries.push(RejectedHeadEvidence { + head_id: head_id.to_owned(), + classification, + reason, + }); + if log.entries.len() > MAX_REJECTED_HEADS { + let remove = log.entries.len() - MAX_REJECTED_HEADS; + log.entries.drain(..remove); + } + let mut random = [0_u8; 8]; + OsRng.fill_bytes(&mut random); + let temporary = directory.join(format!(".rejected-heads-{}.tmp", hex::encode(random))); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary)?; + file.write_all(&serde_json::to_vec_pretty(&log)?)?; + file.sync_all()?; + fs::rename(&temporary, &path)?; + crate::persist::sync_directory(directory)?; + Ok(()) +} + fn git_state_directory(repo: &Path) -> Result { let output = std::process::Command::new("git") .arg("-C") @@ -749,19 +1756,14 @@ fn client_state_path(repo: &Path, remote_name: &str) -> Result Result { match fs::read(path) { Ok(contents) => { - let mut state: ClientState = + let state: ClientState = serde_json::from_slice(&contents).context("parse git-remote-e2ee client state")?; - if state.format_version > 4 { + if state.format_version != 5 { bail!( "unsupported git-remote-e2ee client state format {}", state.format_version ) } - // v2 client states used the observed generation as the import - // checkpoint because observation and import were one operation. - if state.format_version < 3 && state.imported_generation.is_none() { - state.imported_generation = state.generation; - } Ok(state) } Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(ClientState::default()), @@ -769,44 +1771,123 @@ fn read_client_state(path: &Path) -> Result { } } -fn write_client_state( +fn write_client_state_locked( path: &Path, values: &HashSet, head_id: &str, manifest: &Manifest, + verified_refs: &BTreeMap, ) -> Result<()> { let mut values: Vec<_> = values.iter().cloned().collect(); values.sort(); let state = ClientState { - format_version: 4, + format_version: 5, packs: values, head_id: Some(head_id.to_owned()), generation: Some(manifest.generation), imported_generation: Some(manifest.generation), policy_generation: Some(manifest.policy_generation), repository_root: Some(manifest.repository_root.clone()), + verified_refs: verified_refs.clone(), }; - persist_client_state(path, &state) + persist_client_state_locked(path, &state) } -fn write_observed_client_state( +fn write_published_client_state_locked( path: &Path, previous: ClientState, head_id: &str, manifest: &Manifest, ) -> Result<()> { let state = ClientState { - format_version: 4, + format_version: 5, packs: previous.packs, head_id: Some(head_id.to_owned()), generation: Some(manifest.generation), imported_generation: previous.imported_generation, policy_generation: Some(manifest.policy_generation), repository_root: Some(manifest.repository_root.clone()), + verified_refs: previous.verified_refs, }; + persist_client_state_locked(path, &state) +} + +fn persist_client_state_locked(path: &Path, state: &ClientState) -> Result<()> { + let previous = read_client_state(path)?; + let state = merge_client_state(previous, state.clone())?; persist_client_state(path, &state) } +fn merge_client_state(previous: ClientState, mut next: ClientState) -> Result { + if previous.format_version != 5 { + return Ok(next); + } + if next.generation.is_some() + && (next.head_id.is_none() + || next.policy_generation.is_none() + || next.repository_root.is_none()) + { + bail!("client state update is missing a continuity pin field") + } + if let (Some(old_root), Some(new_root)) = (&previous.repository_root, &next.repository_root) + && old_root != new_root + { + bail!("remote repository root changed") + } + if previous.generation.is_some() && next.generation.is_none() { + return Ok(previous); + } + + if let (Some(old_generation), Some(new_generation)) = (previous.generation, next.generation) { + if old_generation > new_generation { + return Ok(previous); + } + if old_generation == new_generation { + if let (Some(old_id), Some(new_id)) = (&previous.head_id, &next.head_id) + && old_id != new_id + { + bail!("remote manifest history forked from the locally pinned head") + } + if let (Some(old_policy), Some(new_policy)) = + (previous.policy_generation, next.policy_generation) + && old_policy != new_policy + { + bail!("remote policy pin changed within the same manifest generation") + } + } + } + if let (Some(old_policy), Some(new_policy)) = + (previous.policy_generation, next.policy_generation) + && new_policy < old_policy + { + bail!("remote policy rolled back") + } + + if previous.repository_root.is_some() { + next.repository_root = previous.repository_root.clone(); + } + if next.head_id.is_none() { + next.head_id = previous.head_id.clone(); + } + if previous.policy_generation.is_some() { + next.policy_generation = previous.policy_generation.max(next.policy_generation); + } + if previous + .imported_generation + .is_some_and(|old| next.imported_generation.is_none_or(|new| new < old)) + { + next.imported_generation = previous.imported_generation; + next.verified_refs = previous.verified_refs; + } + + let mut packs = previous.packs; + packs.extend(next.packs); + packs.sort(); + packs.dedup(); + next.packs = packs; + Ok(next) +} + fn persist_client_state(path: &Path, state: &ClientState) -> Result<()> { let mut random = [0_u8; 8]; OsRng.fill_bytes(&mut random); @@ -842,15 +1923,17 @@ pub(crate) fn testing_write_client_state( crate::persist::create_dir_all_durable(parent)?; } let state = ClientState { - format_version: 4, + format_version: 5, packs: Vec::new(), head_id: Some(head_id.to_owned()), generation: Some(generation), imported_generation: Some(generation), policy_generation: Some(1), repository_root: Some("durability-fixture".to_owned()), + verified_refs: BTreeMap::new(), }; - persist_client_state(path, &state) + let _state_lock = ClientStateLock::acquire(path)?; + persist_client_state_locked(path, &state) } fn validate_remote_name(name: &str) -> Result<()> { @@ -863,3 +1946,115 @@ fn validate_remote_name(name: &str) -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use std::env; + use std::fs; + use std::path::Path; + use std::process::{Child, Command}; + use std::thread; + use std::time::{Duration, Instant}; + + use super::{ClientState, read_client_state, testing_write_client_state}; + + #[test] + fn concurrent_client_state_process_writes_never_lower_or_corrupt_the_floor() { + let temporary = tempfile::tempdir().unwrap(); + let state_path = temporary.path().join("state.json"); + testing_write_client_state(&state_path, "head-10", 10).unwrap(); + let release = temporary.path().join("release-old-writer"); + let ready = temporary.path().join("old-writer-ready"); + let executable = env::current_exe().unwrap(); + + let older = Command::new(&executable) + .args([ + "--exact", + "repository::tests::client_state_process_write_worker", + "--nocapture", + ]) + .env("E2EE_TEST_CLIENT_STATE_PATH", &state_path) + .env("E2EE_TEST_CLIENT_STATE_GENERATION", "11") + .env("E2EE_TEST_CLIENT_STATE_READY", &ready) + .env("E2EE_TEST_CLIENT_STATE_RELEASE", &release) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !ready.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!(ready.exists(), "older writer did not reach its wait point"); + + let newer = launch_client_state_writer(&executable, &state_path, 12); + let newer_output = newer.wait_with_output().unwrap(); + assert!( + newer_output.status.success(), + "newer writer failed: {}", + String::from_utf8_lossy(&newer_output.stderr) + ); + fs::write(&release, "continue").unwrap(); + + let older_output = older.wait_with_output().unwrap(); + assert!( + older_output.status.success(), + "older writer failed: {}", + String::from_utf8_lossy(&older_output.stderr) + ); + let state = read_client_state(&state_path).unwrap(); + assert_eq!(state.generation, Some(12)); + assert_eq!(state.head_id.as_deref(), Some("head-12")); + } + + #[test] + fn client_state_process_write_worker() { + let Ok(path) = env::var("E2EE_TEST_CLIENT_STATE_PATH") else { + return; + }; + let generation = env::var("E2EE_TEST_CLIENT_STATE_GENERATION") + .unwrap() + .parse::() + .unwrap(); + if let (Ok(ready), Ok(release)) = ( + env::var("E2EE_TEST_CLIENT_STATE_READY"), + env::var("E2EE_TEST_CLIENT_STATE_RELEASE"), + ) { + fs::write(&ready, "ready").unwrap(); + let release = Path::new(&release); + let deadline = Instant::now() + Duration::from_secs(10); + while !release.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!(release.exists(), "parent did not release the older writer"); + } + testing_write_client_state(Path::new(&path), &format!("head-{generation}"), generation) + .unwrap(); + } + + fn launch_client_state_writer(executable: &Path, state_path: &Path, generation: u64) -> Child { + Command::new(executable) + .args([ + "--exact", + "repository::tests::client_state_process_write_worker", + "--nocapture", + ]) + .env("E2EE_TEST_CLIENT_STATE_PATH", state_path) + .env("E2EE_TEST_CLIENT_STATE_GENERATION", generation.to_string()) + .spawn() + .unwrap() + } + + #[test] + fn client_state_without_verified_frontier_requires_a_full_check() { + let legacy = br#"{ + "format_version": 4, + "packs": ["pack-id"], + "head_id": "head-id", + "generation": 7, + "imported_generation": 7, + "policy_generation": 2, + "repository_root": "repository-root" + }"#; + let state: ClientState = serde_json::from_slice(legacy).unwrap(); + assert!(state.verified_refs.is_empty()); + } +} diff --git a/src/storage.rs b/src/storage.rs index dff9dbb..3bce390 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,7 +1,8 @@ +use std::collections::HashSet; use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; +use std::io::{BufRead, BufReader, Read, Write}; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Child, ChildStdin, Command, Stdio}; use std::sync::Mutex; use anyhow::{Context, Result, bail}; @@ -11,6 +12,9 @@ use rand::rngs::OsRng; use sha2::{Digest, Sha256}; use thiserror::Error; +use crate::crypto::object_id; +use crate::trace; + #[derive(Debug, Error)] #[error("head changed concurrently (expected {expected:?}, actual {actual:?})")] pub struct CasConflict { @@ -19,11 +23,53 @@ pub struct CasConflict { } const MAX_BUFFERED_OBJECT_SIZE: u64 = 16 * 1024 * 1024; +const MAX_STORAGE_HEAD_BYTES: u64 = 4096; pub trait ObjectStage: Write + Send { fn finish(self: Box, id: &str) -> Result<()>; } +trait ObjectStageSink: Write + Send + Sized { + fn publish(self, id: &str) -> Result<()>; +} + +struct ValidatingObjectStage { + sink: S, + hasher: Sha256, +} + +impl ValidatingObjectStage { + fn wrap(sink: S) -> Box { + Box::new(Self { + sink, + hasher: Sha256::new(), + }) + } +} + +impl Write for ValidatingObjectStage { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let written = self.sink.write(data)?; + self.hasher.update(&data[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.sink.flush() + } +} + +impl ObjectStage for ValidatingObjectStage { + fn finish(self: Box, id: &str) -> Result<()> { + validate_id(id)?; + let Self { sink, hasher } = *self; + if hex::encode(hasher.finalize()) != id { + bail!("staged object hash does not match id") + } + sink.publish(id) + } +} + pub trait Storage { fn begin_object(&self, kind: ObjectKind) -> Result>; fn open_object(&self, kind: ObjectKind, id: &str) -> Result>; @@ -48,13 +94,61 @@ pub trait Storage { fn read_head(&self) -> Result>; fn compare_and_swap_head(&self, expected: Option<&str>, next: &str) -> Result<()>; + + fn observe_head(&self) -> Result { + let head_id = self.read_head()?; + let bytes = head_id.as_deref().unwrap_or_default().as_bytes().to_vec(); + Ok(HeadObservation { + head_id, + token: bytes.clone(), + head_bytes: Some(bytes), + }) + } + + fn compare_and_swap_observed_head(&self, observed: &HeadObservation, next: &str) -> Result<()> { + self.compare_and_swap_head(observed.head_id.as_deref(), next) + } + + fn recovery_history( + &self, + _max_commits: usize, + _max_manifest_bytes: u64, + ) -> Result> { + Ok(None) + } + + fn restore_historical_object(&self, _kind: ObjectKind, _id: &str) -> Result { + Ok(false) + } + + fn prepare_recovery(&self, _manifest_ids: &[String], _pack_ids: &[String]) -> Result<()> { + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub struct HeadObservation { + pub head_id: Option, + pub token: Vec, + pub head_bytes: Option>, +} + +#[derive(Clone, Debug)] +pub struct RecoveryCommit { + pub commit_id: String, + pub head_id: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct RecoveryHistory { + pub commits: Vec, + pub manifests: std::collections::BTreeMap>, } #[derive(Clone, Copy, Debug)] pub enum ObjectKind { Pack, Manifest, - Policy, } impl ObjectKind { @@ -62,7 +156,6 @@ impl ObjectKind { match self { Self::Pack => "objects", Self::Manifest => "manifests", - Self::Policy => "policies", } } } @@ -78,7 +171,7 @@ impl FilesystemStorage { } pub fn initialize(&self) -> Result<()> { - for name in ["objects", "manifests", "policies"] { + for name in ["objects", "manifests"] { let directory = self.root.join(name); crate::persist::create_dir_all_durable(&directory) .with_context(|| format!("create {}", directory.display()))?; @@ -108,14 +201,11 @@ struct FilesystemObjectStage { temporary: PathBuf, root: PathBuf, kind: ObjectKind, - hasher: Sha256, } impl Write for FilesystemObjectStage { fn write(&mut self, data: &[u8]) -> std::io::Result { - let written = self.file.write(data)?; - self.hasher.update(&data[..written]); - Ok(written) + self.file.write(data) } fn flush(&mut self) -> std::io::Result<()> { @@ -123,12 +213,8 @@ impl Write for FilesystemObjectStage { } } -impl ObjectStage for FilesystemObjectStage { - fn finish(mut self: Box, id: &str) -> Result<()> { - validate_id(id)?; - if hex::encode(self.hasher.clone().finalize()) != id { - bail!("staged object hash does not match id") - } +impl ObjectStageSink for FilesystemObjectStage { + fn publish(mut self, id: &str) -> Result<()> { self.file.flush()?; self.file.sync_all()?; crate::persist::durability_checkpoint(crate::persist::STAGE_AFTER_FILE_FLUSH); @@ -184,12 +270,11 @@ impl Storage for FilesystemStorage { .write(true) .create_new(true) .open(&temporary)?; - Ok(Box::new(FilesystemObjectStage { + Ok(ValidatingObjectStage::wrap(FilesystemObjectStage { file, temporary, root: self.root.clone(), kind, - hasher: Sha256::new(), })) } @@ -244,6 +329,87 @@ impl Storage for FilesystemStorage { fs2::FileExt::unlock(&lock)?; Ok(()) } + + fn observe_head(&self) -> Result { + let head_bytes = read_bounded_head_file(&self.root.join("HEAD"))?; + let head_id = head_bytes.as_deref().and_then(|bytes| { + std::str::from_utf8(bytes) + .ok() + .map(str::trim) + .filter(|value| validate_id(value).is_ok()) + .map(ToOwned::to_owned) + }); + Ok(HeadObservation { + head_id, + token: head_bytes.clone().unwrap_or_default(), + head_bytes, + }) + } + + fn compare_and_swap_observed_head(&self, observed: &HeadObservation, next: &str) -> Result<()> { + validate_id(next)?; + let lock = self.lock_file()?; + lock.lock_exclusive()?; + let actual = match fs::read(self.root.join("HEAD")) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => { + fs2::FileExt::unlock(&lock)?; + return Err(error.into()); + } + }; + if actual != observed.head_bytes { + let actual_id = actual.as_deref().and_then(|bytes| { + std::str::from_utf8(bytes) + .ok() + .map(str::trim) + .filter(|value| validate_id(value).is_ok()) + .map(ToOwned::to_owned) + }); + fs2::FileExt::unlock(&lock)?; + return Err(CasConflict { + expected: observed.head_id.clone(), + actual: actual_id, + } + .into()); + } + + write_filesystem_head_locked(&self.root, next)?; + fs2::FileExt::unlock(&lock)?; + Ok(()) + } +} + +fn write_filesystem_head_locked(root: &Path, next: &str) -> Result<()> { + let mut random = [0_u8; 8]; + OsRng.fill_bytes(&mut random); + let temporary = root.join(format!(".HEAD-{}", hex::encode(random))); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary)?; + writeln!(file, "{next}")?; + file.sync_all()?; + crate::persist::durability_checkpoint(crate::persist::STAGE_AFTER_FILE_FLUSH); + fs::rename(&temporary, root.join("HEAD"))?; + crate::persist::durability_checkpoint(crate::persist::STAGE_AFTER_NAME_PUBLISH); + crate::persist::sync_directory(root) + .with_context(|| format!("sync directory {}", root.display()))?; + Ok(()) +} + +fn read_bounded_head_file(path: &Path) -> Result>> { + let mut file = match File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if file.metadata()?.len() > MAX_STORAGE_HEAD_BYTES { + bail!("recovery outer-head byte budget exhausted") + } + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + Ok(Some(bytes)) } fn validate_id(id: &str) -> Result<()> { @@ -275,9 +441,14 @@ fn verify_file_id(path: &Path, id: &str) -> Result<()> { const CARRIER_BRANCH: &str = "git-remote-e2ee"; const CARRIER_CHUNK_SIZE: usize = 32 * 1024 * 1024; +const CARRIER_ATTRIBUTES: &[u8] = b"e2ee/** -delta\n"; +const CARRIER_CACHE_ENV: &str = "GIT_REMOTE_E2EE_CACHE_DIR"; +const CARRIER_CACHE_FETCH_REF: &str = "refs/heads/git-remote-e2ee"; +const MAX_RECOVERY_OUTER_COMMITS: usize = 2048; pub struct GitStorage { checkout: tempfile::TempDir, + remote: String, state: Mutex, } @@ -289,19 +460,27 @@ struct ChunkReader { paths: Vec, next: usize, current: Option, + trace: trace::Io, } impl Read for ChunkReader { fn read(&mut self, output: &mut [u8]) -> std::io::Result { + let started = self.trace.is_active().then(std::time::Instant::now); loop { if let Some(file) = &mut self.current { let read = file.read(output)?; if read != 0 { + self.trace.record( + read, + started.map_or(0, |started| started.elapsed().as_nanos()), + ); return Ok(read); } self.current = None; } if self.next == self.paths.len() { + self.trace + .record(0, started.map_or(0, |started| started.elapsed().as_nanos())); return Ok(0); } self.current = Some(File::open(&self.paths[self.next])?); @@ -310,6 +489,148 @@ impl Read for ChunkReader { } } +struct GitChunkReader { + child: Child, + input: Option, + output: BufReader, + chunks: Vec<(String, String)>, + next: usize, + remaining: u64, + finished: bool, + trace: trace::Io, +} + +impl GitChunkReader { + fn new(repo: &Path, chunks: Vec<(String, String)>) -> Result { + let mut child = carrier_git_command() + .arg("-C") + .arg(repo) + .args(["cat-file", "--batch"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + let input = child.stdin.take().context("open git cat-file stdin")?; + let output = child.stdout.take().context("open git cat-file stdout")?; + Ok(Self { + child, + input: Some(input), + output: BufReader::new(output), + chunks, + next: 0, + remaining: 0, + finished: false, + trace: trace::Io::new("carrier_object_read"), + }) + } + + fn start_next_chunk(&mut self) -> std::io::Result { + if self.next == self.chunks.len() { + return Ok(false); + } + let expected = &self.chunks[self.next].1; + let input = self + .input + .as_mut() + .expect("git cat-file input is open until all chunks are read"); + writeln!(input, "{expected}")?; + input.flush()?; + let mut header = String::new(); + if self.output.read_line(&mut header)? == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "git cat-file ended before returning a carrier chunk", + )); + } + let fields: Vec<_> = header.split_ascii_whitespace().collect(); + if fields.len() != 3 || fields[0] != expected || fields[1] != "blob" { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "git cat-file returned an invalid carrier chunk header", + )); + } + let size = fields[2] + .parse::() + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + let last = self.next + 1 == self.chunks.len(); + if (!last && size != CARRIER_CHUNK_SIZE as u64) + || (last && (size == 0 || size > CARRIER_CHUNK_SIZE as u64)) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "carrier object chunk has invalid size", + )); + } + self.next += 1; + self.remaining = size; + Ok(true) + } + + fn finish(&mut self) -> std::io::Result<()> { + self.input.take(); + let status = self.child.wait()?; + self.finished = true; + if !status.success() { + return Err(std::io::Error::other("git cat-file failed")); + } + Ok(()) + } +} + +impl Read for GitChunkReader { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + if output.is_empty() { + return Ok(0); + } + let started = self.trace.is_active().then(std::time::Instant::now); + if self.remaining == 0 { + if self.next == self.chunks.len() { + self.finish()?; + self.trace + .record(0, started.map_or(0, |started| started.elapsed().as_nanos())); + return Ok(0); + } + if !self.start_next_chunk()? { + return Ok(0); + } + } + let amount = output.len().min(self.remaining as usize); + let read = self.output.read(&mut output[..amount])?; + if read == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "git cat-file truncated a carrier chunk", + )); + } + self.remaining -= read as u64; + if self.remaining == 0 { + let mut trailer = [0_u8; 1]; + self.output.read_exact(&mut trailer)?; + if trailer[0] != b'\n' { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "git cat-file returned a malformed chunk trailer", + )); + } + } + self.trace.record( + read, + started.map_or(0, |started| started.elapsed().as_nanos()), + ); + Ok(read) + } +} + +impl Drop for GitChunkReader { + fn drop(&mut self) { + if !self.finished { + self.input.take(); + let _ = self.child.kill(); + let _ = self.child.wait(); + } + } +} + struct CarrierObjectStage { staging: PathBuf, root: PathBuf, @@ -317,7 +638,7 @@ struct CarrierObjectStage { current: Option, chunk_index: usize, chunk_len: usize, - hasher: Sha256, + trace: trace::Io, } impl CarrierObjectStage { @@ -348,6 +669,7 @@ impl CarrierObjectStage { impl Write for CarrierObjectStage { fn write(&mut self, mut data: &[u8]) -> std::io::Result { let original = data.len(); + let started = self.trace.is_active().then(std::time::Instant::now); while !data.is_empty() { self.open_chunk()?; let available = CARRIER_CHUNK_SIZE - self.chunk_len; @@ -356,13 +678,16 @@ impl Write for CarrierObjectStage { .as_mut() .expect("carrier chunk opened") .write_all(&data[..take])?; - self.hasher.update(&data[..take]); self.chunk_len += take; data = &data[take..]; if self.chunk_len == CARRIER_CHUNK_SIZE { self.finish_chunk()?; } } + self.trace.record( + original, + started.map_or(0, |started| started.elapsed().as_nanos()), + ); Ok(original) } @@ -374,12 +699,8 @@ impl Write for CarrierObjectStage { } } -impl ObjectStage for CarrierObjectStage { - fn finish(mut self: Box, id: &str) -> Result<()> { - validate_id(id)?; - if hex::encode(self.hasher.clone().finalize()) != id { - bail!("staged object hash does not match id") - } +impl ObjectStageSink for CarrierObjectStage { + fn publish(mut self, id: &str) -> Result<()> { self.finish_chunk()?; if self.chunk_index == 0 { bail!("cannot store an empty carrier object") @@ -396,6 +717,7 @@ impl ObjectStage for CarrierObjectStage { paths, next: 0, current: None, + trace: trace::Io::new("carrier_object_read"), })? != id { bail!("object id collision for {id}") @@ -449,46 +771,38 @@ impl GitStorage { if remote.is_empty() { bail!("empty carrier Git remote") } - let checkout = tempfile::Builder::new() + + let lock_timer = trace::Span::new("carrier_cache_lock"); + let mut cache = CarrierCache::lock(remote)?; + drop(lock_timer); + let fetch_timer = trace::Span::new("carrier_cache_fetch"); + let mut base_commit = cache.refresh(remote)?; + drop(fetch_timer); + let mut checkout = tempfile::Builder::new() .prefix("git-remote-e2ee-carrier-") .tempdir()?; - git_command( - checkout - .path() - .parent() - .context("carrier tempdir has no parent")?, - &[ - "clone", - "--quiet", - "--no-checkout", - remote, - checkout.path_str()?, - ], - )?; - git_command(checkout.path(), &["config", "user.name", "git-remote-e2ee"])?; - git_command( - checkout.path(), - &["config", "user.email", "git-remote-e2ee@invalid"], - )?; - - let remote_ref = format!("refs/remotes/origin/{CARRIER_BRANCH}"); - let base_commit = git_rev_parse(checkout.path(), &remote_ref)?; - match &base_commit { - Some(_) => { - git_command( - checkout.path(), - &["checkout", "--quiet", "-B", CARRIER_BRANCH, &remote_ref], - )?; - } - None => { - git_command( - checkout.path(), - &["checkout", "--quiet", "--orphan", CARRIER_BRANCH], - )?; + let checkout_timer = trace::Span::new("carrier_checkout_create"); + if let Err(error) = + create_carrier_checkout(checkout.path(), &cache.path, remote, &base_commit) + { + if !cache.is_corrupt()? { + return Err(error).context("create carrier checkout"); } + drop(checkout); + cache.rebuild()?; + base_commit = cache.refresh(remote)?; + checkout = tempfile::Builder::new() + .prefix("git-remote-e2ee-carrier-") + .tempdir()?; + create_carrier_checkout(checkout.path(), &cache.path, remote, &base_commit) + .context("create carrier checkout after rebuilding corrupt cache")?; } + drop(checkout_timer); + drop(cache); + Ok(Self { checkout, + remote: remote.to_owned(), state: Mutex::new(GitStorageState { base_commit }), }) } @@ -515,26 +829,33 @@ impl GitStorage { validate_id(&value)?; Ok(Some(value)) } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let Some(commit) = git_rev_parse(self.root(), "HEAD")? else { + return Ok(None); + }; + let Some(bytes) = self.head_bytes_at_commit(&commit)? else { + return Ok(None); + }; + let value = String::from_utf8(bytes)?.trim().to_owned(); + validate_id(&value)?; + Ok(Some(value)) + } Err(error) => Err(error.into()), } } - fn remote_tip(&self) -> Result> { - let output = carrier_git_command() - .arg("-C") - .arg(self.root()) - .args(["ls-remote", "--heads", "origin"]) - .arg(format!("refs/heads/{CARRIER_BRANCH}")) - .output()?; - if !output.status.success() { - bail!( - "git ls-remote failed: {}", - String::from_utf8_lossy(&output.stderr).trim() - ) + fn local_head_bytes(&self) -> Result>> { + let path = self.root().join("e2ee/HEAD"); + match read_bounded_head_file(&path) { + Ok(Some(bytes)) => Ok(Some(bytes)), + Ok(None) => { + let Some(commit) = git_rev_parse(self.root(), "HEAD")? else { + return Ok(None); + }; + self.head_bytes_at_commit(&commit) + } + Err(error) => Err(error), } - let text = String::from_utf8(output.stdout)?; - Ok(text.split_whitespace().next().map(ToOwned::to_owned)) } fn head_at_commit(&self, commit: Option<&str>) -> Result> { @@ -553,17 +874,194 @@ impl GitStorage { validate_id(&value)?; Ok(Some(value)) } -} -trait TempDirPath { - fn path_str(&self) -> Result<&str>; -} + fn head_bytes_at_commit(&self, commit: &str) -> Result>> { + let object = format!("{commit}:e2ee/HEAD"); + let size = carrier_git_command() + .arg("-C") + .arg(self.root()) + .args(["cat-file", "-s", &object]) + .output()?; + if !size.status.success() { + return Ok(None); + } + let size = String::from_utf8(size.stdout)?.trim().parse::()?; + if size > MAX_STORAGE_HEAD_BYTES { + bail!("recovery outer-head byte budget exhausted") + } + let output = carrier_git_command() + .arg("-C") + .arg(self.root()) + .args(["show", &object]) + .output()?; + if output.status.success() { + Ok(Some(output.stdout)) + } else { + Ok(None) + } + } -impl TempDirPath for tempfile::TempDir { - fn path_str(&self) -> Result<&str> { - self.path() - .to_str() - .context("carrier checkout path is not UTF-8") + fn commit_history(&self, limit: usize) -> Result> { + let maximum = limit.saturating_add(1).to_string(); + let output = carrier_git_command() + .arg("-C") + .arg(self.root()) + .args([ + "rev-list", + "--parents", + &format!("--max-count={maximum}"), + "HEAD", + ]) + .output()?; + ensure_git_success(&output, "git rev-list carrier history")?; + // Every publication is a single-parent fast-forward commit, so + // legitimate carrier history is linear. A merge can only come from a + // storage-level writer and can hide a legitimate state behind a + // non-first parent, so recovery refuses to interpret such history. + let mut commits = Vec::new(); + for line in String::from_utf8(output.stdout)?.lines() { + let mut fields = line.split_ascii_whitespace(); + let commit = fields.next().context("parse carrier history entry")?; + if fields.count() > 1 { + bail!( + "carrier history contains merge commit {commit}; recovery requires linear carrier history and cannot rule out a hidden legitimate state" + ) + } + commits.push(commit.to_owned()); + } + Ok(commits) + } + + fn chunk_entries_at( + &self, + commit: &str, + kind: ObjectKind, + id: &str, + ) -> Result>> { + carrier_chunk_entries(self.root(), commit, kind, id) + } + + fn object_at_commit( + &self, + commit: &str, + kind: ObjectKind, + id: &str, + max_bytes: u64, + ) -> Result>> { + let Some(chunks) = self.chunk_entries_at(commit, kind, id)? else { + return Ok(None); + }; + let mut bytes = Vec::new(); + for (index, (path, _)) in chunks.iter().enumerate() { + let output = carrier_git_command() + .arg("-C") + .arg(self.root()) + .args(["show", &format!("{commit}:{path}")]) + .output()?; + ensure_git_success(&output, "git show historical carrier object")?; + let length = output.stdout.len(); + if (index + 1 < chunks.len() && length != CARRIER_CHUNK_SIZE) + || (index + 1 == chunks.len() && (length == 0 || length > CARRIER_CHUNK_SIZE)) + { + bail!("historical carrier object chunk has invalid size") + } + let next_length = (bytes.len() as u64) + .checked_add(length as u64) + .context("historical carrier object size overflow")?; + if next_length > max_bytes { + bail!("carrier recovery manifest byte budget exhausted") + } + bytes.extend_from_slice(&output.stdout); + } + if object_id(&bytes) != id { + bail!("historical carrier object content id mismatch") + } + Ok(Some(bytes)) + } + + fn restore_object_at(&self, commit: &str, kind: ObjectKind, id: &str) -> Result { + let Some(chunks) = self.chunk_entries_at(commit, kind, id)? else { + return Ok(false); + }; + let staging = tempfile::Builder::new() + .prefix("git-remote-e2ee-restore-") + .tempdir_in(self.root())?; + let object_dir = staging.path().join("object"); + fs::create_dir(&object_dir)?; + let mut hasher = Sha256::new(); + for (index, (path, _)) in chunks.iter().enumerate() { + let output = carrier_git_command() + .arg("-C") + .arg(self.root()) + .args(["show", &format!("{commit}:{path}")]) + .output()?; + ensure_git_success(&output, "git show historical carrier object")?; + let length = output.stdout.len(); + if (index + 1 < chunks.len() && length != CARRIER_CHUNK_SIZE) + || (index + 1 == chunks.len() && (length == 0 || length > CARRIER_CHUNK_SIZE)) + { + bail!("historical carrier object chunk has invalid size") + } + hasher.update(&output.stdout); + let chunk_path = object_dir.join(format!("{index:08}")); + let mut file = File::create(chunk_path)?; + file.write_all(&output.stdout)?; + file.sync_all()?; + } + if hex::encode(hasher.finalize()) != id { + bail!("historical carrier object content id mismatch") + } + let target = self.object_directory(kind, id)?; + if target.exists() { + fs::remove_dir_all(&target)?; + } + fs::create_dir_all( + target + .parent() + .context("carrier object path has no parent")?, + )?; + fs::rename(object_dir, target)?; + Ok(true) + } + + fn publish_checkout_head( + &self, + next: &str, + expected: Option, + cache: &mut CarrierCache, + ) -> Result<()> { + fs::create_dir_all(self.root().join("e2ee"))?; + fs::write(self.root().join("e2ee/HEAD"), format!("{next}\n"))?; + let add_timer = trace::Span::new("carrier_git_add"); + stage_carrier_checkout(self.root(), &cache.path.join("objects"))?; + drop(add_timer); + let commit_timer = trace::Span::new("carrier_git_commit"); + git_command_with_object_directory( + self.root(), + &cache.path.join("objects"), + &["commit", "--quiet", "-m", "git-remote-e2ee storage update"], + )?; + let new_commit = + git_rev_parse(self.root(), "HEAD")?.context("carrier commit was not created")?; + drop(commit_timer); + let push_timer = trace::Span::new("carrier_git_push"); + let push = carrier_git_command() + .arg("-C") + .arg(self.root()) + .args(["push", "--quiet", "origin"]) + .arg(format!("HEAD:refs/heads/{CARRIER_BRANCH}")) + .output()?; + if !push.status.success() { + let latest = cache.refresh(&self.remote)?; + return Err(CasConflict { + expected, + actual: self.head_at_commit(latest.as_deref())?, + } + .into()); + } + drop(push_timer); + cache.update_published_tip(&new_commit)?; + Ok(()) } } @@ -573,27 +1071,40 @@ impl Storage for GitStorage { OsRng.fill_bytes(&mut random); let staging = self.root().join(format!(".stage-{}", hex::encode(random))); fs::create_dir(&staging)?; - Ok(Box::new(CarrierObjectStage { + Ok(ValidatingObjectStage::wrap(CarrierObjectStage { staging, root: self.root().to_path_buf(), kind, current: None, chunk_index: 0, chunk_len: 0, - hasher: Sha256::new(), + trace: trace::Io::new("carrier_stage_write"), })) } fn open_object(&self, kind: ObjectKind, id: &str) -> Result> { - let paths = validated_chunk_paths(&self.object_directory(kind, id)?)?; - Ok(Box::new(ChunkReader { - paths, - next: 0, - current: None, - })) + let directory = self.object_directory(kind, id)?; + if directory.exists() { + let paths = validated_chunk_paths(&directory)?; + return Ok(Box::new(ChunkReader { + paths, + next: 0, + current: None, + trace: trace::Io::new("carrier_object_read"), + })); + } + let Some(commit) = git_rev_parse(self.root(), "HEAD")? else { + bail!("carrier object {id} is unavailable") + }; + let chunks = self + .chunk_entries_at(&commit, kind, id)? + .with_context(|| format!("carrier object {id} is unavailable"))?; + Ok(Box::new(GitChunkReader::new(self.root(), chunks)?)) } fn read_head(&self) -> Result> { + // `open` refreshed the remote before constructing this checkout; CAS + // refreshes again before trusting this snapshot for a write. self.local_head() } @@ -603,7 +1114,10 @@ impl Storage for GitStorage { .state .lock() .map_err(|_| anyhow::anyhow!("carrier state lock poisoned"))?; - let remote_tip = self.remote_tip()?; + let mut cache = CarrierCache::lock(&self.remote)?; + let refresh_timer = trace::Span::new("carrier_cas_refresh"); + let remote_tip = cache.refresh(&self.remote)?; + drop(refresh_timer); if remote_tip != state.base_commit { return Err(CasConflict { expected: expected.map(ToOwned::to_owned), @@ -622,13 +1136,19 @@ impl Storage for GitStorage { fs::create_dir_all(self.root().join("e2ee"))?; fs::write(self.root().join("e2ee/HEAD"), format!("{next}\n"))?; - git_command(self.root(), &["add", "e2ee"])?; - git_command( + let add_timer = trace::Span::new("carrier_git_add"); + stage_carrier_checkout(self.root(), &cache.path.join("objects"))?; + drop(add_timer); + let commit_timer = trace::Span::new("carrier_git_commit"); + git_command_with_object_directory( self.root(), + &cache.path.join("objects"), &["commit", "--quiet", "-m", "git-remote-e2ee storage update"], )?; + drop(commit_timer); let new_commit = git_rev_parse(self.root(), "HEAD")?.context("carrier commit was not created")?; + let push_timer = trace::Span::new("carrier_git_push"); let push = carrier_git_command() .arg("-C") .arg(self.root()) @@ -636,16 +1156,605 @@ impl Storage for GitStorage { .arg(format!("HEAD:refs/heads/{CARRIER_BRANCH}")) .output()?; if !push.status.success() { - let latest = self.remote_tip()?; + let latest = cache.refresh(&self.remote)?; return Err(CasConflict { expected: expected.map(ToOwned::to_owned), actual: self.head_at_commit(latest.as_deref())?, } .into()); } + drop(push_timer); + cache.update_published_tip(&new_commit)?; state.base_commit = Some(new_commit); Ok(()) } + + fn observe_head(&self) -> Result { + let state = self + .state + .lock() + .map_err(|_| anyhow::anyhow!("carrier state lock poisoned"))?; + let head_bytes = self.local_head_bytes()?; + let head_id = head_bytes.as_deref().and_then(|bytes| { + std::str::from_utf8(bytes) + .ok() + .map(str::trim) + .filter(|value| validate_id(value).is_ok()) + .map(ToOwned::to_owned) + }); + Ok(HeadObservation { + head_id, + token: state + .base_commit + .as_deref() + .unwrap_or_default() + .as_bytes() + .to_vec(), + head_bytes, + }) + } + + fn compare_and_swap_observed_head(&self, observed: &HeadObservation, next: &str) -> Result<()> { + validate_id(next)?; + let mut state = self + .state + .lock() + .map_err(|_| anyhow::anyhow!("carrier state lock poisoned"))?; + let expected_tip = state.base_commit.clone(); + if observed.token != expected_tip.as_deref().unwrap_or_default().as_bytes() { + bail!("recovery storage token does not match the opened carrier tip") + } + let mut cache = CarrierCache::lock(&self.remote)?; + let remote_tip = cache.refresh(&self.remote)?; + if remote_tip != expected_tip { + return Err(CasConflict { + expected: observed.head_id.clone(), + actual: self.head_at_commit(remote_tip.as_deref())?, + } + .into()); + } + let actual_bytes = self.local_head_bytes()?; + if actual_bytes != observed.head_bytes { + return Err(CasConflict { + expected: observed.head_id.clone(), + actual: actual_bytes.as_deref().and_then(|bytes| { + std::str::from_utf8(bytes) + .ok() + .map(str::trim) + .filter(|value| validate_id(value).is_ok()) + .map(ToOwned::to_owned) + }), + } + .into()); + } + self.publish_checkout_head(next, observed.head_id.clone(), &mut cache)?; + let new_commit = + git_rev_parse(self.root(), "HEAD")?.context("carrier commit was not created")?; + state.base_commit = Some(new_commit); + Ok(()) + } + + fn recovery_history( + &self, + max_commits: usize, + max_manifest_bytes: u64, + ) -> Result> { + let commits = self.commit_history(max_commits)?; + if commits.len() > max_commits { + bail!("carrier recovery outer-commit budget exhausted") + } + let mut history = RecoveryHistory::default(); + let mut total_bytes = 0_u64; + for commit_id in commits { + let head_id = self + .head_bytes_at_commit(&commit_id)? + .as_deref() + .and_then(|bytes| { + std::str::from_utf8(bytes) + .ok() + .map(str::trim) + .filter(|value| validate_id(value).is_ok()) + .map(ToOwned::to_owned) + }); + if let Some(id) = &head_id + && !history.manifests.contains_key(id) + { + let remaining = max_manifest_bytes.saturating_sub(total_bytes); + if let Some(bytes) = + self.object_at_commit(&commit_id, ObjectKind::Manifest, id, remaining)? + { + total_bytes = total_bytes + .checked_add(bytes.len() as u64) + .context("carrier recovery manifest byte count overflow")?; + if total_bytes > max_manifest_bytes { + bail!("carrier recovery manifest byte budget exhausted") + } + history.manifests.insert(id.clone(), bytes); + } + } + history.commits.push(RecoveryCommit { commit_id, head_id }); + } + Ok(Some(history)) + } + + fn restore_historical_object(&self, kind: ObjectKind, id: &str) -> Result { + let commits = self.commit_history(MAX_RECOVERY_OUTER_COMMITS)?; + if commits.len() > MAX_RECOVERY_OUTER_COMMITS { + bail!("carrier recovery outer-commit budget exhausted") + } + for commit in commits { + if self.restore_object_at(&commit, kind, id)? { + return Ok(true); + } + } + Ok(false) + } + + fn prepare_recovery(&self, manifest_ids: &[String], pack_ids: &[String]) -> Result<()> { + let manifests: HashSet<&str> = manifest_ids.iter().map(String::as_str).collect(); + let packs: HashSet<&str> = pack_ids.iter().map(String::as_str).collect(); + retain_carrier_objects(&self.root().join("e2ee/manifests"), &manifests)?; + retain_carrier_objects(&self.root().join("e2ee/objects"), &packs)?; + remove_unretained_index_objects(self.root(), ObjectKind::Manifest, &manifests)?; + remove_unretained_index_objects(self.root(), ObjectKind::Pack, &packs)?; + Ok(()) + } +} + +fn remove_unretained_index_objects( + repo: &Path, + kind: ObjectKind, + keep: &HashSet<&str>, +) -> Result<()> { + let tree = format!("e2ee/{}", kind.directory()); + let output = carrier_git_command() + .arg("-C") + .arg(repo) + .args(["ls-files", "-z", "--", &tree]) + .output()?; + ensure_git_success(&output, "git ls-files carrier recovery objects")?; + let mut remove = Vec::new(); + for path in output + .stdout + .split(|byte| *byte == 0) + .filter(|path| !path.is_empty()) + { + let path = std::str::from_utf8(path)?; + let mut fields = path.split('/'); + if fields.next() != Some("e2ee") || fields.next() != Some(kind.directory()) { + bail!("unexpected carrier index path during recovery") + } + let _prefix = fields.next().context("carrier index path has no prefix")?; + let id = fields + .next() + .context("carrier index path has no object id")?; + let _chunk = fields.next().context("carrier index path has no chunk")?; + if fields.next().is_some() { + bail!("unexpected carrier index path depth during recovery") + } + if !keep.contains(id) { + remove.extend_from_slice(path.as_bytes()); + remove.push(0); + } + } + if !remove.is_empty() { + git_command_with_input( + repo, + &["update-index", "--force-remove", "-z", "--stdin"], + &remove, + )?; + } + Ok(()) +} + +fn retain_carrier_objects(root: &Path, keep: &HashSet<&str>) -> Result<()> { + let entries = match fs::read_dir(root) { + Ok(entries) => entries.collect::>>()?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + for prefix in entries { + if !prefix.file_type()?.is_dir() { + fs::remove_file(prefix.path())?; + continue; + } + let objects = fs::read_dir(prefix.path())?.collect::>>()?; + for object in objects { + let id = object.file_name(); + if id.to_str().is_none_or(|id| !keep.contains(id)) { + if object.file_type()?.is_dir() { + fs::remove_dir_all(object.path())?; + } else { + fs::remove_file(object.path())?; + } + } + } + if fs::read_dir(prefix.path())?.next().is_none() { + fs::remove_dir(prefix.path())?; + } + } + Ok(()) +} + +fn carrier_chunk_entries( + repo: &Path, + commit: &str, + kind: ObjectKind, + id: &str, +) -> Result>> { + validate_id(id)?; + let directory = format!("e2ee/{}/{}/{id}", kind.directory(), &id[..2]); + let output = carrier_git_command() + .arg("-C") + .arg(repo) + .args(["ls-tree", "-r", "-z", commit, "--", &directory]) + .output()?; + ensure_git_success(&output, "git ls-tree carrier object")?; + if output.stdout.is_empty() { + return Ok(None); + } + if output.stdout.len() > 8 * 1024 * 1024 { + bail!("carrier object listing exceeds its size budget") + } + let mut chunks = Vec::new(); + for entry in output + .stdout + .split(|byte| *byte == 0) + .filter(|entry| !entry.is_empty()) + { + let tab = entry + .iter() + .position(|byte| *byte == b'\t') + .context("parse carrier tree entry")?; + let (metadata, path) = (&entry[..tab], &entry[tab + 1..]); + let metadata = std::str::from_utf8(metadata)?; + let path = std::str::from_utf8(path)?.to_owned(); + let mut fields = metadata.split_ascii_whitespace(); + if fields.next() != Some("100644") || fields.next() != Some("blob") { + bail!("carrier object contains a non-regular blob") + } + let blob_id = fields.next().context("carrier blob has no id")?; + if fields.next().is_some() || !path.starts_with(&format!("{directory}/")) { + bail!("unexpected carrier object path") + } + chunks.push((path, blob_id.to_owned())); + } + chunks.sort_by(|left, right| left.0.cmp(&right.0)); + if chunks.is_empty() || chunks.len() > 4096 { + bail!("carrier object has an invalid chunk count") + } + for (index, (path, _)) in chunks.iter().enumerate() { + let expected = format!("{directory}/{index:08}"); + if path != &expected { + bail!("carrier object chunks are not a dense canonical sequence") + } + } + Ok(Some(chunks)) +} + +struct CarrierCache { + remote_dir: PathBuf, + path: PathBuf, + _lock: File, +} + +impl CarrierCache { + fn lock(remote: &str) -> Result { + let normalized = normalize_carrier_remote(remote)?; + let hash = hex::encode(Sha256::digest(normalized.as_bytes())); + let root = carrier_cache_root()?; + fs::create_dir_all(&root) + .with_context(|| format!("create carrier cache directory {}", root.display()))?; + let root = fs::canonicalize(&root)?; + let lock_path = root.join(format!(".{hash}.lock")); + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&lock_path) + .with_context(|| format!("open carrier cache lock {}", lock_path.display()))?; + let wait_timer = trace::Span::new("carrier_cache_lock_wait"); + lock.lock_exclusive() + .with_context(|| format!("lock carrier cache {}", lock_path.display()))?; + drop(wait_timer); + let remote_dir = root.join(&hash); + Ok(Self { + remote_dir, + path: PathBuf::new(), + _lock: lock, + }) + } + + fn refresh(&mut self, remote: &str) -> Result> { + let initialize_timer = trace::Span::new("carrier_cache_initialize"); + self.ensure_initialized()?; + drop(initialize_timer); + let fetch_timer = trace::Span::new("carrier_cache_fetch_remote"); + match fetch_carrier_branch(&self.path, remote) { + Ok(tip) => { + drop(fetch_timer); + Ok(tip) + } + Err(fetch_error) if self.is_corrupt()? => { + self.rebuild()?; + let result = fetch_carrier_branch(&self.path, remote).with_context(|| { + format!( + "fetch carrier branch after rebuilding corrupt cache (initial fetch failed: {fetch_error:#})" + ) + }); + drop(fetch_timer); + result + } + Err(fetch_error) => { + drop(fetch_timer); + Err(fetch_error) + } + } + } + + fn ensure_initialized(&mut self) -> Result<()> { + fs::create_dir_all(&self.remote_dir).with_context(|| { + format!( + "create per-remote carrier cache {}", + self.remote_dir.display() + ) + })?; + let current = self.remote_dir.join("current"); + let generation = fs::read_to_string(¤t) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| valid_cache_generation(value)); + if let Some(generation) = generation { + self.path = self.remote_dir.join(generation); + } + let usable = !self.path.as_os_str().is_empty() + && self.path.exists() + && cache_git_output(&self.path, &["rev-parse", "--is-bare-repository"]).is_ok_and( + |output| { + output.status.success() + && String::from_utf8_lossy(&output.stdout).trim() == "true" + }, + ); + if !usable { + self.initialize_generation()?; + } + if configure_cache(&self.path).is_err() { + self.rebuild()?; + } + Ok(()) + } + + fn initialize_generation(&mut self) -> Result<()> { + fs::create_dir_all(&self.remote_dir)?; + let mut random = [0_u8; 8]; + OsRng.fill_bytes(&mut random); + let generation = format!("objects-{}-{}", std::process::id(), hex::encode(random)); + self.path = self.remote_dir.join(generation); + let output = carrier_git_command() + .args(["init", "--bare", "--quiet"]) + .arg(&self.path) + .output()?; + ensure_git_success(&output, "git init --bare carrier cache")?; + configure_cache(&self.path)?; + cache_git_command( + &self.path, + &["symbolic-ref", "HEAD", CARRIER_CACHE_FETCH_REF], + )?; + self.publish_current() + } + + fn is_corrupt(&self) -> Result { + let output = cache_git_output(&self.path, &["fsck", "--full"])?; + Ok(!output.status.success()) + } + + fn rebuild(&mut self) -> Result<()> { + // Existing temporary checkouts may still borrow the previous object + // directory. Keep that generation intact and publish a fresh one. + self.initialize_generation() + } + + fn publish_current(&self) -> Result<()> { + let generation = self + .path + .file_name() + .context("carrier cache generation has no name")?; + let generation = generation + .to_str() + .context("carrier cache generation name is not UTF-8")?; + let mut random = [0_u8; 8]; + OsRng.fill_bytes(&mut random); + let temporary = self.remote_dir.join(format!( + ".current-{}-{}", + std::process::id(), + hex::encode(random) + )); + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary)?; + writeln!(file, "{generation}")?; + file.sync_all()?; + let current = self.remote_dir.join("current"); + if current.exists() { + fs::remove_file(¤t)?; + } + fs::rename(&temporary, ¤t)?; + Ok(()) + } + + fn update_published_tip(&self, commit: &str) -> Result<()> { + cache_git_command(&self.path, &["update-ref", CARRIER_CACHE_FETCH_REF, commit]) + } +} + +fn valid_cache_generation(value: &str) -> bool { + value.starts_with("objects-") + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + +fn carrier_cache_root() -> Result { + if let Some(override_dir) = std::env::var_os(CARRIER_CACHE_ENV) { + if override_dir.is_empty() { + bail!("{CARRIER_CACHE_ENV} must not be empty") + } + return Ok(PathBuf::from(override_dir)); + } + let cache_home = match std::env::var_os("XDG_CACHE_HOME") { + Some(path) if !path.is_empty() => PathBuf::from(path), + _ => PathBuf::from( + std::env::var_os("HOME").context("HOME is unset and XDG_CACHE_HOME is unset")?, + ) + .join(".cache"), + }; + Ok(cache_home.join("git-remote-e2ee").join("carrier")) +} + +fn normalize_carrier_remote(remote: &str) -> Result { + if remote.trim().is_empty() { + bail!("empty carrier Git remote") + } + if remote.contains("://") { + return Ok(remote.trim().trim_end_matches('/').to_owned()); + } + if remote == remote.trim() + && let Some((host, path)) = remote.split_once(':') + && !host.is_empty() + && !host.contains('/') + && !host.contains('\\') + && !(host.len() == 1 && host.as_bytes()[0].is_ascii_alphabetic()) + && !remote.starts_with("./") + && !remote.starts_with("../") + && !remote.starts_with('/') + { + return Ok(format!("{host}:{}", path.trim_end_matches('/'))); + } + + let path = Path::new(remote); + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + match fs::canonicalize(&absolute) { + Ok(path) => Ok(path.to_string_lossy().into_owned()), + Err(_) => Ok(lexically_normalize(&absolute) + .to_string_lossy() + .into_owned()), + } +} + +fn lexically_normalize(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + normalized.pop(); + } + component => normalized.push(component.as_os_str()), + } + } + normalized +} + +fn configure_cache(cache: &Path) -> Result<()> { + cache_git_command(cache, &["config", "gc.auto", "0"])?; + cache_git_command(cache, &["config", "maintenance.auto", "false"])?; + configure_git_pack_settings(|key, value| cache_git_command(cache, &["config", key, value]))?; + Ok(()) +} + +fn configure_git_pack_settings(mut configure: impl FnMut(&str, &str) -> Result<()>) -> Result<()> { + for (key, value) in [ + ("core.compression", "0"), + ("core.looseCompression", "0"), + ("pack.compression", "0"), + ("pack.window", "0"), + ("pack.depth", "0"), + ("core.bigFileThreshold", "1m"), + ] { + configure(key, value)?; + } + Ok(()) +} + +fn fetch_carrier_branch(cache: &Path, remote: &str) -> Result> { + let refspec = format!("+refs/heads/{CARRIER_BRANCH}:{CARRIER_CACHE_FETCH_REF}"); + let fetch = cache_git_output(cache, &["fetch", "--quiet", "--no-tags", remote, &refspec])?; + if fetch.status.success() { + return git_rev_parse(cache, CARRIER_CACHE_FETCH_REF); + } + + let ls_remote = cache_git_output( + cache, + &["ls-remote", "--heads", remote, CARRIER_CACHE_FETCH_REF], + )?; + ensure_git_success(&ls_remote, "git ls-remote carrier branch")?; + if String::from_utf8_lossy(&ls_remote.stdout).trim().is_empty() { + let delete = cache_git_output(cache, &["update-ref", "-d", CARRIER_CACHE_FETCH_REF])?; + ensure_git_success(&delete, "git update-ref delete missing carrier branch")?; + return Ok(None); + } + ensure_git_success(&fetch, "git fetch carrier branch")?; + unreachable!("failed git fetch must return an error") +} + +fn create_carrier_checkout( + checkout: &Path, + cache: &Path, + remote: &str, + base_commit: &Option, +) -> Result<()> { + let _timer = trace::Span::new("carrier_checkout_git_setup"); + let output = carrier_git_command() + .arg("-C") + .arg(checkout) + .args(["init", "--quiet"]) + .output()?; + ensure_git_success(&output, "git init carrier checkout")?; + let cache_objects = cache.join("objects"); + let cache_objects = cache_objects + .to_str() + .context("carrier cache object path is not UTF-8")?; + if cache_objects.contains('\n') { + bail!("carrier cache object path contains a newline") + } + fs::write( + checkout.join(".git/objects/info/alternates"), + format!("{cache_objects}\n"), + )?; + git_command(checkout, &["config", "gc.auto", "0"])?; + git_command(checkout, &["config", "maintenance.auto", "false"])?; + configure_git_pack_settings(|key, value| git_command(checkout, &["config", key, value]))?; + git_command(checkout, &["config", "user.name", "git-remote-e2ee"])?; + git_command( + checkout, + &["config", "user.email", "git-remote-e2ee@invalid"], + )?; + git_command(checkout, &["remote", "add", "origin", remote])?; + + let remote_ref = format!("refs/remotes/origin/{CARRIER_BRANCH}"); + match base_commit { + Some(commit) => { + git_command(checkout, &["update-ref", &remote_ref, commit])?; + let local_ref = format!("refs/heads/{CARRIER_BRANCH}"); + git_command(checkout, &["update-ref", &local_ref, commit])?; + git_command(checkout, &["symbolic-ref", "HEAD", &local_ref])?; + git_command(checkout, &["read-tree", commit])?; + } + None => { + git_command( + checkout, + &["checkout", "--quiet", "--orphan", CARRIER_BRANCH], + )?; + } + } + fs::write(checkout.join(".gitattributes"), CARRIER_ATTRIBUTES)?; + Ok(()) } fn git_rev_parse(repo: &Path, reference: &str) -> Result> { @@ -661,6 +1770,29 @@ fn git_rev_parse(repo: &Path, reference: &str) -> Result> { } } +fn cache_git_command(cache: &Path, args: &[&str]) -> Result<()> { + let output = cache_git_output(cache, args)?; + ensure_git_success(&output, &format!("git {}", args.join(" "))) +} + +fn cache_git_output(cache: &Path, args: &[&str]) -> Result { + Ok(carrier_git_command() + .arg("--git-dir") + .arg(cache) + .args(args) + .output()?) +} + +fn ensure_git_success(output: &std::process::Output, action: &str) -> Result<()> { + if !output.status.success() { + bail!( + "{action} failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + } + Ok(()) +} + fn git_command(repo: &Path, args: &[&str]) -> Result<()> { let output = carrier_git_command() .arg("-C") @@ -677,6 +1809,126 @@ fn git_command(repo: &Path, args: &[&str]) -> Result<()> { Ok(()) } +fn git_command_with_input(repo: &Path, args: &[&str], input: &[u8]) -> Result<()> { + let mut child = carrier_git_command() + .arg("-C") + .arg(repo) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + child + .stdin + .take() + .context("open git stdin")? + .write_all(input)?; + let output = child.wait_with_output()?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + } + Ok(()) +} + +fn stage_carrier_checkout(repo: &Path, object_directory: &Path) -> Result<()> { + fn collect(root: &Path, path: &Path, output: &mut Vec) -> Result<()> { + let entry = fs::symlink_metadata(path)?; + if entry.file_type().is_dir() { + for child in fs::read_dir(path)? { + collect(root, &child?.path(), output)?; + } + } else if entry.file_type().is_file() { + let relative = path + .strip_prefix(root)? + .to_str() + .context("carrier checkout path is not UTF-8")?; + if relative.contains('\n') || relative.contains('\0') { + bail!("carrier checkout path contains a forbidden character") + } + output.extend_from_slice(relative.as_bytes()); + output.push(0); + } else { + bail!("carrier checkout contains a non-regular staged path") + } + Ok(()) + } + + let mut paths = Vec::new(); + let attributes = repo.join(".gitattributes"); + if attributes.exists() { + collect(repo, &attributes, &mut paths)?; + } + let encrypted = repo.join("e2ee"); + if encrypted.exists() { + collect(repo, &encrypted, &mut paths)?; + } + if paths.is_empty() { + bail!("carrier publication has no staged files") + } + git_command_with_object_directory_input( + repo, + object_directory, + &["add", "-A", "--pathspec-from-file=-", "--pathspec-file-nul"], + &paths, + ) +} + +fn git_command_with_object_directory( + repo: &Path, + object_directory: &Path, + args: &[&str], +) -> Result<()> { + let output = carrier_git_command() + .arg("-C") + .arg(repo) + .env("GIT_OBJECT_DIRECTORY", object_directory) + .args(args) + .output()?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + } + Ok(()) +} + +fn git_command_with_object_directory_input( + repo: &Path, + object_directory: &Path, + args: &[&str], + input: &[u8], +) -> Result<()> { + let mut child = carrier_git_command() + .arg("-C") + .arg(repo) + .env("GIT_OBJECT_DIRECTORY", object_directory) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + child + .stdin + .take() + .context("open git pathspec stdin")? + .write_all(input)?; + let output = child.wait_with_output()?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + } + Ok(()) +} + fn carrier_git_command() -> Command { let mut command = Command::new("git"); for variable in [ @@ -695,12 +1947,54 @@ fn carrier_git_command() -> Command { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Barrier}; use std::thread; use super::*; use crate::crypto::object_id; + struct PartialWriteSink { + published: Arc, + } + + impl Write for PartialWriteSink { + fn write(&mut self, data: &[u8]) -> std::io::Result { + Ok(data.len().min(3)) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl ObjectStageSink for PartialWriteSink { + fn publish(self, _id: &str) -> Result<()> { + self.published.store(true, Ordering::Relaxed); + Ok(()) + } + } + + #[test] + fn object_stage_validates_all_written_bytes_before_publishing() { + let data = b"partial writes must all contribute to the staged object hash"; + let published = Arc::new(AtomicBool::new(false)); + let mut stage = ValidatingObjectStage::wrap(PartialWriteSink { + published: Arc::clone(&published), + }); + stage.write_all(data).unwrap(); + stage.finish(&object_id(data)).unwrap(); + assert!(published.load(Ordering::Relaxed)); + + let published = Arc::new(AtomicBool::new(false)); + let mut stage = ValidatingObjectStage::wrap(PartialWriteSink { + published: Arc::clone(&published), + }); + stage.write_all(data).unwrap(); + assert!(stage.finish(&object_id(b"different bytes")).is_err()); + assert!(!published.load(Ordering::Relaxed)); + } + #[test] fn head_compare_and_swap_rejects_stale_writer() { let directory = tempfile::tempdir().unwrap(); @@ -791,7 +2085,8 @@ mod tests { reader_id(ChunkReader { paths, next: 0, - current: None + current: None, + trace: trace::Io::new("carrier_object_read"), }) .unwrap(), object_id(b"only") diff --git a/src/trace.rs b/src/trace.rs new file mode 100644 index 0000000..bce9f66 --- /dev/null +++ b/src/trace.rs @@ -0,0 +1,110 @@ +use std::sync::OnceLock; +use std::time::Instant; + +fn enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var("GIT_REMOTE_E2EE_TRACE").is_ok_and(|value| value == "1")) +} + +pub(crate) fn report_chacha20_backend() { + if !enabled() { + return; + } + + static REPORTED: OnceLock<()> = OnceLock::new(); + REPORTED.get_or_init(|| { + eprintln!( + "git-remote-e2ee trace chacha20_backend={}", + chacha20_backend() + ); + }); +} + +pub(crate) fn count(name: &'static str, value: usize) { + if enabled() { + eprintln!("git-remote-e2ee trace count={name} value={value}"); + } +} + +#[allow(unexpected_cfgs)] +fn chacha20_backend() -> &'static str { + if cfg!(chacha20_backend = "soft") { + "portable" + } else if cfg!(all(target_arch = "aarch64", target_feature = "neon")) { + "aarch64-neon" + } else if cfg!(target_arch = "aarch64") { + "aarch64-portable" + } else if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + "x86-runtime-dispatch" + } else { + "portable" + } +} + +pub(crate) struct Span { + name: &'static str, + started: Option, +} + +impl Span { + pub(crate) fn new(name: &'static str) -> Self { + Self { + name, + started: enabled().then(Instant::now), + } + } +} + +impl Drop for Span { + fn drop(&mut self) { + if let Some(started) = self.started { + eprintln!( + "git-remote-e2ee trace phase={} elapsed_ms={}", + self.name, + started.elapsed().as_millis() + ); + } + } +} + +pub(crate) struct Io { + name: &'static str, + started: Option, + elapsed_nanos: u128, + bytes: u64, +} + +impl Io { + pub(crate) fn new(name: &'static str) -> Self { + Self { + name, + started: enabled().then(Instant::now), + elapsed_nanos: 0, + bytes: 0, + } + } + + pub(crate) fn record(&mut self, bytes: usize, elapsed_nanos: u128) { + if self.started.is_some() { + self.bytes = self.bytes.saturating_add(bytes as u64); + self.elapsed_nanos = self.elapsed_nanos.saturating_add(elapsed_nanos); + } + } + + pub(crate) fn is_active(&self) -> bool { + self.started.is_some() + } +} + +impl Drop for Io { + fn drop(&mut self) { + if self.started.is_some() { + eprintln!( + "git-remote-e2ee trace io={} elapsed_ms={} bytes={}", + self.name, + self.elapsed_nanos / 1_000_000, + self.bytes + ); + } + } +} diff --git a/tests/anonymous_recipients.rs b/tests/anonymous_recipients.rs new file mode 100644 index 0000000..215529e --- /dev/null +++ b/tests/anonymous_recipients.rs @@ -0,0 +1,222 @@ +use std::env; +use std::fs; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::Mutex; + +use base64::Engine; +use git_remote_e2ee::crypto::{KeyFile, object_id}; +use git_remote_e2ee::manifest::{ManifestHeader, peek_manifest_header}; +use git_remote_e2ee::policy::DeviceRoles; +use git_remote_e2ee::repository::EncryptedRepository; +use git_remote_e2ee::storage::{FilesystemStorage, GitStorage, Storage}; + +static CACHE_ENV_LOCK: Mutex<()> = Mutex::new(()); + +#[test] +fn filesystem_storage_hides_device_metadata_and_pads_reader_envelopes() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path().join("storage"); + run_metadata_scenario( + || FilesystemStorage::new(&root), + || filesystem_snapshot(&root), + || assert!(!root.join("policies").exists()), + temporary.path(), + ); +} + +#[test] +fn carrier_history_hides_device_metadata_and_pads_reader_envelopes() { + let temporary = tempfile::tempdir().unwrap(); + let _cache_lock = CACHE_ENV_LOCK.lock().unwrap(); + unsafe { + env::set_var( + "GIT_REMOTE_E2EE_CACHE_DIR", + temporary.path().join("carrier-cache"), + ) + }; + let carrier = temporary.path().join("carrier.git"); + let output = Command::new("git") + .args(["init", "--bare", "-q"]) + .arg(&carrier) + .output() + .unwrap(); + assert!( + output.status.success(), + "git init --bare: {}", + String::from_utf8_lossy(&output.stderr) + ); + run_metadata_scenario( + || GitStorage::open(carrier.to_str().unwrap()).unwrap(), + || carrier_snapshot(&carrier), + || { + let paths = git_output(&carrier, &["log", "--all", "--name-only", "--format="]); + assert!(!String::from_utf8_lossy(&paths.stdout).contains("e2ee/policies/")); + }, + temporary.path(), + ); +} + +fn run_metadata_scenario( + make_storage: impl Fn() -> S, + snapshot: impl Fn() -> Vec, + no_policy_objects: impl Fn(), + scratch: &Path, +) { + let owner = KeyFile::generate(); + let pin = scratch.join("owner.admin-state.json"); + let repository = EncryptedRepository::new(make_storage(), owner.clone()); + repository.initialize().unwrap(); + repository.pin_admin_state(&pin).unwrap(); + let mut devices = vec![owner.public_device().unwrap()]; + assert_eq!( + stored_header(make_storage()).generation_key_envelopes.len(), + 4 + ); + assert_storage_is_anonymous(&snapshot(), &devices); + no_policy_objects(); + + let readers: Vec<_> = (0..4) + .map(|_| KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap()) + .collect(); + for (index, reader) in readers.iter().enumerate() { + repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::collaborator(), + &pin, + ) + .unwrap(); + devices.push(reader.public_device().unwrap()); + let expected = if index == 3 { 8 } else { 4 }; + assert_eq!( + stored_header(make_storage()).generation_key_envelopes.len(), + expected + ); + assert_storage_is_anonymous(&snapshot(), &devices); + no_policy_objects(); + } + + repository + .revoke_device(&readers[0].device_id().unwrap(), &pin) + .unwrap(); + assert_eq!( + stored_header(make_storage()).generation_key_envelopes.len(), + 4 + ); + assert_storage_is_anonymous(&snapshot(), &devices); + no_policy_objects(); +} + +fn stored_header(storage: impl Storage) -> ManifestHeader { + let head = storage.read_head().unwrap().unwrap(); + let bytes = storage + .get_object(git_remote_e2ee::storage::ObjectKind::Manifest, &head) + .unwrap(); + assert_eq!(object_id(&bytes), head); + peek_manifest_header(&bytes).unwrap() +} + +fn assert_storage_is_anonymous(bytes: &[u8], devices: &[git_remote_e2ee::crypto::PublicDevice]) { + let mut tokens = vec![ + b"reader".to_vec(), + b"writer".to_vec(), + b"administrator".to_vec(), + b"signing_public_key".to_vec(), + b"wrapping_public_key".to_vec(), + b"device_id".to_vec(), + b"policies/".to_vec(), + ]; + for device in devices { + tokens.push(device.device_id.as_bytes().to_vec()); + tokens.push(hex::decode(&device.device_id).unwrap()); + tokens.push(device.signing_public_key.as_bytes().to_vec()); + tokens.push( + base64::engine::general_purpose::STANDARD + .decode(&device.signing_public_key) + .unwrap(), + ); + tokens.push(device.wrapping_public_key.as_bytes().to_vec()); + tokens.push( + base64::engine::general_purpose::STANDARD + .decode(&device.wrapping_public_key) + .unwrap(), + ); + } + for token in tokens { + assert!( + !bytes.windows(token.len()).any(|window| window == token), + "storage contains plaintext metadata token {:?}", + String::from_utf8_lossy(&token) + ); + } +} + +fn filesystem_snapshot(root: &Path) -> Vec { + let mut bytes = Vec::new(); + append_files(root, root, &mut bytes); + bytes +} + +fn carrier_snapshot(root: &Path) -> Vec { + let mut bytes = Vec::new(); + append_files(root, root, &mut bytes); + + let reachable = git_output(root, &["rev-list", "--objects", "--all"]); + assert!( + reachable.status.success(), + "git rev-list: {}", + String::from_utf8_lossy(&reachable.stderr) + ); + let ids = String::from_utf8(reachable.stdout).unwrap(); + let mut child = Command::new("git") + .arg(format!("--git-dir={}", root.display())) + .args(["cat-file", "--batch"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + { + let stdin = child.stdin.as_mut().unwrap(); + for line in ids.lines() { + let id = line.split_whitespace().next().unwrap(); + writeln!(stdin, "{id}").unwrap(); + } + } + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "git cat-file: {}", + String::from_utf8_lossy(&output.stderr) + ); + bytes.extend_from_slice(&output.stdout); + bytes +} + +fn append_files(root: &Path, directory: &Path, bytes: &mut Vec) { + let mut entries: Vec<_> = fs::read_dir(directory) + .unwrap() + .map(Result::unwrap) + .collect(); + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let relative = path.strip_prefix(root).unwrap(); + bytes.extend_from_slice(relative.to_string_lossy().as_bytes()); + if entry.file_type().unwrap().is_dir() { + append_files(root, &path, bytes); + } else { + bytes.extend_from_slice(&fs::read(&path).unwrap()); + } + } +} + +fn git_output(root: &Path, args: &[&str]) -> std::process::Output { + Command::new("git") + .arg(format!("--git-dir={}", root.display())) + .args(args) + .output() + .unwrap() +} diff --git a/tests/carrier_git.rs b/tests/carrier_git.rs index c115d68..d4c7d53 100644 --- a/tests/carrier_git.rs +++ b/tests/carrier_git.rs @@ -1,13 +1,57 @@ +use std::collections::BTreeMap; use std::env; use std::fs; use std::path::Path; use std::process::{Command, Output}; -use std::sync::{Arc, Barrier}; +use std::sync::{Arc, Barrier, Mutex, MutexGuard}; use std::thread; +use std::time::{Duration, Instant}; -use git_remote_e2ee::crypto::KeyFile; -use git_remote_e2ee::repository::EncryptedRepository; -use git_remote_e2ee::storage::{CasConflict, GitStorage}; +use git_remote_e2ee::crypto::{KeyFile, object_id, random_key}; +use git_remote_e2ee::manifest::{ + Manifest, ManifestAuthorization, open_manifest, peek_manifest_header, seal_manifest, + unwrap_generation_key, wrap_predecessor_key, +}; +use git_remote_e2ee::policy::{DeviceRoles, PolicyState}; +use git_remote_e2ee::repository::{EncryptedRepository, RecoveryClass, RecoveryOptions}; +use git_remote_e2ee::storage::{CasConflict, GitStorage, ObjectKind, Storage}; + +static CACHE_ENV_LOCK: Mutex<()> = Mutex::new(()); + +fn isolated_cache(temporary: &tempfile::TempDir) -> MutexGuard<'static, ()> { + let guard = CACHE_ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let cache = temporary.path().join("carrier-cache"); + // This test binary serializes all tests that open GitStorage, keeping the + // process-global override stable while helper subprocesses inherit it. + unsafe { env::set_var("GIT_REMOTE_E2EE_CACHE_DIR", cache) }; + guard +} + +fn cache_repo(cache_root: &Path) -> std::path::PathBuf { + let per_remote = fs::read_dir(cache_root) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.is_dir()) + .expect("one per-remote carrier cache directory"); + let generation = fs::read_to_string(per_remote.join("current")).unwrap(); + per_remote.join(generation.trim()) +} + +fn reachable_object_count(repo: &Path) -> usize { + let output = Command::new("git") + .arg(format!("--git-dir={}", repo.display())) + .args(["rev-list", "--objects", "--all"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "git rev-list failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().lines().count() +} fn git_output(repo: &Path, args: &[&str], with_helper: bool) -> Output { let mut command = Command::new("git"); @@ -59,6 +103,107 @@ fn init_carrier(root: &Path, carrier: &Path, key: &KeyFile) { .unwrap(); } +fn create_reader_signed_junk( + storage: &GitStorage, + reader: &KeyFile, + policy: &PolicyState, + parent_id: &str, +) -> String { + let encrypted = storage.get_object(ObjectKind::Manifest, parent_id).unwrap(); + let header = peek_manifest_header(&encrypted).unwrap(); + let parent_key = unwrap_generation_key(&header, reader).unwrap(); + let opened = open_manifest(&encrypted, &parent_key).unwrap(); + let parent = opened.manifest(); + let generation_key = random_key(); + let manifest = Manifest { + format_version: parent.format_version, + repository_root: parent.repository_root.clone(), + generation: parent.generation + 1, + previous: Some(parent_id.to_owned()), + policy_id: parent.policy_id.clone(), + policy_generation: parent.policy_generation, + authorization: ManifestAuthorization::Writer, + total_pack_count: parent.total_pack_count, + refs: parent.refs.clone(), + new_packs: Vec::new(), + predecessor_key_wrap: Some( + wrap_predecessor_key( + &generation_key, + &parent_key, + &parent.repository_root, + parent.generation + 1, + parent_id, + ) + .unwrap(), + ), + introduced_policy: None, + }; + let encrypted = seal_manifest( + reader, + policy, + &generation_key, + ManifestAuthorization::Writer, + manifest, + ) + .unwrap(); + let id = object_id(&encrypted); + storage + .put_object_if_absent(ObjectKind::Manifest, &id, &encrypted) + .unwrap(); + id +} + +fn create_writer_successor( + storage: &GitStorage, + writer: &KeyFile, + policy: &PolicyState, + parent_id: &str, + refs: BTreeMap, +) -> String { + let encrypted = storage.get_object(ObjectKind::Manifest, parent_id).unwrap(); + let header = peek_manifest_header(&encrypted).unwrap(); + let parent_key = unwrap_generation_key(&header, writer).unwrap(); + let opened = open_manifest(&encrypted, &parent_key).unwrap(); + let parent = opened.manifest(); + let generation_key = random_key(); + let manifest = Manifest { + format_version: parent.format_version, + repository_root: parent.repository_root.clone(), + generation: parent.generation + 1, + previous: Some(parent_id.to_owned()), + policy_id: parent.policy_id.clone(), + policy_generation: parent.policy_generation, + authorization: ManifestAuthorization::Writer, + total_pack_count: parent.total_pack_count, + refs, + new_packs: Vec::new(), + predecessor_key_wrap: Some( + wrap_predecessor_key( + &generation_key, + &parent_key, + &parent.repository_root, + parent.generation + 1, + parent_id, + ) + .unwrap(), + ), + introduced_policy: None, + }; + let encrypted = seal_manifest( + writer, + policy, + &generation_key, + ManifestAuthorization::Writer, + manifest, + ) + .unwrap(); + let id = object_id(&encrypted); + storage + .put_object_if_absent(ObjectKind::Manifest, &id, &encrypted) + .unwrap(); + id +} + fn add_helper_to_path(command: &mut Command) { let helper = Path::new(env!("CARGO_BIN_EXE_git-remote-e2ee")); let mut paths = vec![helper.parent().unwrap().to_path_buf()]; @@ -69,6 +214,7 @@ fn add_helper_to_path(command: &mut Command) { #[test] fn carrier_git_supports_native_push_and_clone_without_plaintext() { let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); let carrier = temporary.path().join("carrier.git"); let source = temporary.path().join("source"); let clone = temporary.path().join("clone"); @@ -92,6 +238,14 @@ fn carrier_git_supports_native_push_and_clone_without_plaintext() { false, ); git(&source, &["push", "private", "main"], true); + assert_eq!( + git( + &carrier, + &["show", "refs/heads/git-remote-e2ee:.gitattributes"], + false, + ), + "e2ee/** -delta" + ); let mut clone_command = Command::new("git"); clone_command @@ -138,9 +292,108 @@ fn carrier_git_supports_native_push_and_clone_without_plaintext() { assert_eq!(grep.status.code(), Some(1)); } +#[test] +fn carrier_git_supports_tags_and_ref_deletion() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + let source = temporary.path().join("source"); + let returning = temporary.path().join("returning"); + let key_path = temporary.path().join("repository.key.json"); + let main = init_source(&source, "carrier refs"); + let key = KeyFile::generate(); + key.write_new(&key_path).unwrap(); + init_carrier(temporary.path(), &carrier, &key); + + let url = format!("e2ee::git+{}", carrier.display()); + git(&source, &["remote", "add", "private", &url], false); + git( + &source, + &[ + "config", + "remote.private.e2ee-key", + key_path.to_str().unwrap(), + ], + false, + ); + git(&source, &["push", "private", "main"], true); + git( + &source, + &["tag", "-a", "v-carrier", "-m", "carrier release", &main], + false, + ); + git(&source, &["push", "private", "v-carrier"], true); + + let mut clone_command = Command::new("git"); + clone_command + .args([ + "-c", + &format!("e2ee.key={}", key_path.display()), + "clone", + "-q", + &url, + ]) + .arg(&returning); + add_helper_to_path(&mut clone_command); + let clone_output = clone_command.output().unwrap(); + assert!( + clone_output.status.success(), + "clone stdout={} stderr={}", + String::from_utf8_lossy(&clone_output.stdout), + String::from_utf8_lossy(&clone_output.stderr) + ); + assert_eq!( + git( + &returning, + &["cat-file", "-t", "refs/tags/v-carrier"], + false + ), + "tag" + ); + + git(&source, &["switch", "-q", "-c", "doomed"], false); + fs::write(source.join("note.md"), "branch to delete\n").unwrap(); + git(&source, &["add", "note.md"], false); + git(&source, &["commit", "-q", "-m", "doomed"], false); + git(&source, &["push", "private", "doomed"], true); + git(&source, &["push", "--delete", "private", "doomed"], true); + git(&source, &["push", "private", ":refs/tags/v-carrier"], true); + git( + &returning, + &["fetch", "--prune", "--prune-tags", "origin"], + true, + ); + + assert!( + !git_output( + &returning, + &["show-ref", "--verify", "refs/remotes/origin/doomed"], + false + ) + .status + .success() + ); + assert!( + !git_output( + &returning, + &["show-ref", "--verify", "refs/tags/v-carrier"], + false + ) + .status + .success() + ); + let manifest = + EncryptedRepository::new(GitStorage::open(carrier.to_str().unwrap()).unwrap(), key) + .verify() + .unwrap(); + assert!(!manifest.refs.contains_key("refs/heads/doomed")); + assert!(!manifest.refs.contains_key("refs/tags/v-carrier")); +} + #[test] fn carrier_git_allows_exactly_one_concurrent_writer() { let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); let carrier = temporary.path().join("carrier.git"); let source_a = temporary.path().join("source-a"); let source_b = temporary.path().join("source-b"); @@ -203,9 +456,283 @@ fn carrier_git_allows_exactly_one_concurrent_writer() { ); } +#[test] +fn carrier_cache_fetches_only_new_objects() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + let first = "1".repeat(64); + let second = "2".repeat(64); + + let initial = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + initial.compare_and_swap_head(None, &first).unwrap(); + drop(initial); + + let cached = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let cache_root = temporary.path().join("carrier-cache"); + let cache = cache_repo(&cache_root); + let before_push = reachable_object_count(&cache); + let remote_before = reachable_object_count(&carrier); + assert_eq!(before_push, remote_before); + + cached.compare_and_swap_head(Some(&first), &second).unwrap(); + drop(cached); + let remote_after = reachable_object_count(&carrier); + assert!(remote_after > remote_before); + let after_push = reachable_object_count(&cache); + assert_eq!(after_push, remote_after); + + let updated = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + assert_eq!(after_push - before_push, remote_after - remote_before); + assert_eq!(after_push, remote_after); + drop(updated); + + let unchanged = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + assert_eq!(reachable_object_count(&cache), after_push); + drop(unchanged); +} + +#[test] +fn carrier_cache_does_not_hide_remote_access_errors() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let missing_remote = temporary.path().join("missing-carrier.git"); + let error = match GitStorage::open(missing_remote.to_str().unwrap()) { + Ok(_) => panic!("opening a missing carrier unexpectedly succeeded"), + Err(error) => error, + }; + assert!( + error.to_string().contains("fetch") || error.to_string().contains("ls-remote"), + "unexpected remote error: {error:#}" + ); +} + +#[test] +fn stale_carrier_cache_is_refreshed_before_compare_and_swap() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + let stale = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let winner = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let winner_head = "3".repeat(64); + let stale_head = "4".repeat(64); + + winner.compare_and_swap_head(None, &winner_head).unwrap(); + let error = stale.compare_and_swap_head(None, &stale_head).unwrap_err(); + let conflict = error.downcast_ref::().unwrap(); + assert_eq!(conflict.actual.as_deref(), Some(winner_head.as_str())); + + let remote_head = git( + temporary.path(), + &[ + &format!("--git-dir={}", carrier.display()), + "show", + "refs/heads/git-remote-e2ee:e2ee/HEAD", + ], + false, + ); + assert_eq!(remote_head, winner_head); +} + +#[test] +fn deleted_and_corrupt_carrier_caches_are_rebuilt() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + let head = "5".repeat(64); + let storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + storage.compare_and_swap_head(None, &head).unwrap(); + drop(storage); + + let populated = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + assert_eq!( + populated.read_head().unwrap().as_deref(), + Some(head.as_str()) + ); + drop(populated); + let cache_root = temporary.path().join("carrier-cache"); + let cache = cache_repo(&cache_root); + + fs::remove_dir_all(&cache).unwrap(); + let rebuilt = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + assert_eq!(rebuilt.read_head().unwrap().as_deref(), Some(head.as_str())); + drop(rebuilt); + + let cache = cache_repo(&cache_root); + let object_listing = git(&cache, &["rev-list", "--objects", "--all"], false); + let object_id = object_listing + .lines() + .next() + .and_then(|line| line.split_whitespace().next()) + .expect("fetched carrier commit"); + let loose_object = cache + .join("objects") + .join(&object_id[..2]) + .join(&object_id[2..]); + if loose_object.exists() { + // Unlink first because local Git transports may hard-link loose objects. + fs::remove_file(&loose_object).unwrap(); + fs::write(&loose_object, b"corrupt cache object").unwrap(); + } else { + let pack = fs::read_dir(cache.join("objects/pack")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.extension() + .is_some_and(|extension| extension == "pack") + }) + .expect("fetched carrier objects are packed"); + fs::write(pack, b"corrupt cache pack").unwrap(); + } + + let recovered = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + assert_eq!( + recovered.read_head().unwrap().as_deref(), + Some(head.as_str()) + ); + assert_eq!( + reachable_object_count(&cache_repo(&cache_root)), + reachable_object_count(&carrier) + ); +} + +#[test] +fn carrier_git_process_race_has_exactly_one_winner() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + let ready_a = temporary.path().join("ready-a"); + let ready_b = temporary.path().join("ready-b"); + let result_a = temporary.path().join("result-a"); + let result_b = temporary.path().join("result-b"); + let start = temporary.path().join("start"); + let executable = env::current_exe().unwrap(); + let launch = |candidate: &str, ready: &Path, result: &Path| { + Command::new(&executable) + .args([ + "--ignored", + "--exact", + "carrier_git_process_race_worker", + "--nocapture", + ]) + .env("E2EE_TEST_RACE_REMOTE", &carrier) + .env("E2EE_TEST_RACE_CANDIDATE", candidate) + .env("E2EE_TEST_RACE_READY", ready) + .env("E2EE_TEST_RACE_RESULT", result) + .env("E2EE_TEST_RACE_START", &start) + .spawn() + .unwrap() + }; + let first = launch(&"6".repeat(64), &ready_a, &result_a); + let second = launch(&"7".repeat(64), &ready_b, &result_b); + + let deadline = Instant::now() + Duration::from_secs(10); + while (!ready_a.exists() || !ready_b.exists()) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!( + ready_a.exists() && ready_b.exists(), + "race workers did not become ready" + ); + fs::write(&start, "go").unwrap(); + + let first_output = first.wait_with_output().unwrap(); + let second_output = second.wait_with_output().unwrap(); + assert!( + first_output.status.success(), + "first worker failed: {}", + String::from_utf8_lossy(&first_output.stderr) + ); + assert!( + second_output.status.success(), + "second worker failed: {}", + String::from_utf8_lossy(&second_output.stderr) + ); + let results = [ + fs::read_to_string(result_a).unwrap(), + fs::read_to_string(result_b).unwrap(), + ]; + assert_eq!( + results + .iter() + .filter(|result| result.as_str() == "winner") + .count(), + 1 + ); + assert_eq!( + results + .iter() + .filter(|result| result.as_str() == "conflict") + .count(), + 1 + ); + assert_eq!( + git( + temporary.path(), + &[ + &format!("--git-dir={}", carrier.display()), + "rev-list", + "--count", + "refs/heads/git-remote-e2ee", + ], + false, + ), + "1" + ); +} + +#[test] +#[ignore = "launched as a separate process by carrier_git_process_race_has_exactly_one_winner"] +fn carrier_git_process_race_worker() { + let Ok(remote) = env::var("E2EE_TEST_RACE_REMOTE") else { + return; + }; + let candidate = env::var("E2EE_TEST_RACE_CANDIDATE").unwrap(); + let ready = env::var("E2EE_TEST_RACE_READY").unwrap(); + let result = env::var("E2EE_TEST_RACE_RESULT").unwrap(); + let start = env::var("E2EE_TEST_RACE_START").unwrap(); + let storage = GitStorage::open(&remote).unwrap(); + fs::write(ready, "ready").unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while !Path::new(&start).exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!( + Path::new(&start).exists(), + "parent did not release CAS race" + ); + let outcome = match storage.compare_and_swap_head(None, &candidate) { + Ok(()) => "winner", + Err(error) if error.downcast_ref::().is_some() => "conflict", + Err(error) => panic!("unexpected CAS race failure: {error:#}"), + }; + fs::write(result, outcome).unwrap(); +} + #[test] fn native_fetch_rejects_outer_carrier_rollback() { let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); let carrier = temporary.path().join("carrier.git"); let source = temporary.path().join("source"); let clone = temporary.path().join("clone"); @@ -272,6 +799,7 @@ fn native_fetch_rejects_outer_carrier_rollback() { #[test] fn carrier_git_detects_tampered_ciphertext_chunk() { let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); let carrier = temporary.path().join("carrier.git"); let source = temporary.path().join("source"); let attacker = temporary.path().join("attacker"); @@ -324,9 +852,727 @@ fn carrier_git_detects_tampered_ciphertext_chunk() { ); } +#[test] +fn carrier_recovery_chooses_legitimate_descendant_and_restores_deleted_ciphertext() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + let source = temporary.path().join("source"); + let returning = temporary.path().join("returning"); + let fresh = temporary.path().join("fresh"); + let cache_root = temporary.path().join("carrier-cache"); + init_source(&source, "legitimate descendant content"); + init_source(&returning, "returning local content"); + init_source(&fresh, "fresh local content"); + + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + let repository = EncryptedRepository::new( + GitStorage::open(carrier.to_str().unwrap()).unwrap(), + owner.clone(), + ); + repository.initialize().unwrap(); + repository.pin_admin_state(&admin_pin).unwrap(); + let (reader_policy_manifest, _) = repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + let policy_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let policy_encrypted = policy_storage + .get_object(ObjectKind::Manifest, &reader_policy_manifest) + .unwrap(); + let policy_header = peek_manifest_header(&policy_encrypted).unwrap(); + let policy_key = unwrap_generation_key(&policy_header, &reader).unwrap(); + let policy_manifest = open_manifest(&policy_encrypted, &policy_key).unwrap(); + let floor_policy = PolicyState::parse( + policy_manifest + .manifest() + .introduced_policy + .as_deref() + .unwrap(), + ) + .unwrap(); + let pushed_object = git(&source, &["rev-parse", "HEAD"], false); + repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + let (floor_id, floor_manifest) = repository.current_manifest().unwrap(); + let pack_id = floor_manifest.new_packs[0].id.clone(); + repository.fetch_into(&returning, "private").unwrap(); + let floor_generation: u64 = serde_json::from_slice::( + &fs::read(returning.join(".git/git-remote-e2ee/private/state.json")).unwrap(), + ) + .unwrap()["generation"] + .as_u64() + .unwrap(); + assert_eq!(floor_manifest.generation, floor_generation); + repository + .revoke_device(&reader.device_id().unwrap(), &admin_pin) + .unwrap(); + let (legitimate_id, legitimate) = repository.current_manifest().unwrap(); + assert_eq!(legitimate.refs["refs/heads/main"], pushed_object); + + let attacker = temporary.path().join("carrier-edit"); + git( + temporary.path(), + &[ + "clone", + "-q", + "-b", + "git-remote-e2ee", + carrier.to_str().unwrap(), + attacker.to_str().unwrap(), + ], + false, + ); + git(&attacker, &["config", "user.name", "Carrier editor"], false); + git( + &attacker, + &["config", "user.email", "carrier-editor@example.invalid"], + false, + ); + fs::remove_dir_all( + attacker + .join("e2ee/objects") + .join(&pack_id[..2]) + .join(&pack_id), + ) + .unwrap(); + fs::write(attacker.join("e2ee/HEAD"), format!("{floor_id}\n")).unwrap(); + git(&attacker, &["add", "-A", "e2ee"], false); + git( + &attacker, + &["commit", "-q", "-m", "replay floor and delete pack"], + false, + ); + git( + &attacker, + &["push", "-q", "origin", "git-remote-e2ee"], + false, + ); + + let replay_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let junk_id = create_reader_signed_junk(&replay_storage, &reader, &floor_policy, &floor_id); + replay_storage + .compare_and_swap_head(Some(&floor_id), &junk_id) + .unwrap(); + + let recovery_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let recoverer = EncryptedRepository::new(recovery_storage, owner.clone()); + let report = recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: false, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert_eq!(report.classification, RecoveryClass::Invalid); + assert_eq!(report.default_base.as_deref(), Some(legitimate_id.as_str())); + assert!(report.replays.contains(&floor_id)); + assert!( + report + .candidates + .iter() + .any(|candidate| candidate.manifest_id == legitimate_id) + ); + + let older_base = recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: true, + base: Some(&floor_id), + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert!( + older_base + .blocked_reason + .as_deref() + .unwrap() + .contains("--discard-newer") + ); + + let recovered = recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + let recovered_id = recovered.published_manifest.unwrap(); + let (current_id, manifest) = recoverer.current_manifest().unwrap(); + assert_eq!(current_id, recovered_id); + assert_eq!(manifest.previous.as_deref(), Some(legitimate_id.as_str())); + assert_eq!(manifest.generation, floor_generation + 2); + assert_eq!(manifest.refs["refs/heads/main"], pushed_object); + assert!( + GitStorage::open(carrier.to_str().unwrap()) + .unwrap() + .open_object(ObjectKind::Pack, &pack_id) + .is_ok() + ); + let recovered_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let recovered_bytes = recovered_storage + .get_object(ObjectKind::Manifest, ¤t_id) + .unwrap(); + let recovered_header = peek_manifest_header(&recovered_bytes).unwrap(); + assert!(unwrap_generation_key(&recovered_header, &reader).is_err()); + + recoverer.fetch_into(&returning, "private").unwrap(); + recoverer.fetch_into(&fresh, "private").unwrap(); + assert_eq!( + git( + &fresh, + &["show", "refs/remotes/private/main:note.md"], + false + ), + "legitimate descendant content" + ); + let carrier_head = Command::new("git") + .arg(format!("--git-dir={}", carrier.display())) + .args(["show", "refs/heads/git-remote-e2ee:e2ee/HEAD"]) + .output() + .unwrap(); + assert!(carrier_head.status.success()); + assert_eq!( + String::from_utf8(carrier_head.stdout).unwrap().trim(), + current_id + ); + assert!(cache_root.exists()); +} + +#[test] +fn carrier_recovery_offers_hidden_legitimate_content_descendant() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + let source = temporary.path().join("source"); + let returning = temporary.path().join("returning"); + let fresh = temporary.path().join("fresh"); + init_source(&source, "legitimate descendant content"); + init_source(&returning, "returning local content"); + init_source(&fresh, "fresh local content"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + let repository = EncryptedRepository::new( + GitStorage::open(carrier.to_str().unwrap()).unwrap(), + owner.clone(), + ); + repository.initialize().unwrap(); + repository.pin_admin_state(&admin_pin).unwrap(); + let (floor_id, _) = repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + let floor_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let floor_bytes = floor_storage + .get_object(ObjectKind::Manifest, &floor_id) + .unwrap(); + let floor_header = peek_manifest_header(&floor_bytes).unwrap(); + let floor_key = unwrap_generation_key(&floor_header, &reader).unwrap(); + let floor_opened = open_manifest(&floor_bytes, &floor_key).unwrap(); + let floor_policy = PolicyState::parse( + floor_opened + .manifest() + .introduced_policy + .as_deref() + .unwrap(), + ) + .unwrap(); + repository.fetch_into(&returning, "private").unwrap(); + + let pushed_object = git(&source, &["rev-parse", "HEAD"], false); + repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + let (legitimate_id, legitimate) = repository.current_manifest().unwrap(); + assert_eq!(legitimate.refs["refs/heads/main"], pushed_object); + + let attacker = temporary.path().join("carrier-edit"); + git( + temporary.path(), + &[ + "clone", + "-q", + "-b", + "git-remote-e2ee", + carrier.to_str().unwrap(), + attacker.to_str().unwrap(), + ], + false, + ); + git(&attacker, &["config", "user.name", "Carrier editor"], false); + git( + &attacker, + &["config", "user.email", "carrier-editor@example.invalid"], + false, + ); + fs::write(attacker.join("e2ee/HEAD"), format!("{floor_id}\n")).unwrap(); + git(&attacker, &["add", "-A", "e2ee"], false); + git(&attacker, &["commit", "-q", "-m", "replay floor"], false); + git( + &attacker, + &["push", "-q", "origin", "git-remote-e2ee"], + false, + ); + + let replay_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let junk_id = create_reader_signed_junk(&replay_storage, &reader, &floor_policy, &floor_id); + replay_storage + .compare_and_swap_head(Some(&floor_id), &junk_id) + .unwrap(); + + let recoverer = + EncryptedRepository::new(GitStorage::open(carrier.to_str().unwrap()).unwrap(), owner); + let report = recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: false, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert_eq!(report.classification, RecoveryClass::Invalid); + assert_eq!(report.default_base.as_deref(), Some(legitimate_id.as_str())); + assert!(report.replays.contains(&floor_id)); + assert!(report.candidates.iter().any(|candidate| { + candidate.manifest_id == legitimate_id && candidate.generation == legitimate.generation + })); + + let stale_choice = recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: true, + base: Some(&floor_id), + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert!( + stale_choice + .blocked_reason + .as_deref() + .unwrap() + .contains("--discard-newer") + ); + + let recovered = recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + let recovered_id = recovered.published_manifest.unwrap(); + let (head, manifest) = recoverer.current_manifest().unwrap(); + assert_eq!(head, recovered_id); + assert_eq!(manifest.previous.as_deref(), Some(legitimate_id.as_str())); + assert_eq!(manifest.refs["refs/heads/main"], pushed_object); + recoverer.fetch_into(&returning, "private").unwrap(); + recoverer.fetch_into(&fresh, "private").unwrap(); + assert_eq!( + git( + &fresh, + &["show", "refs/remotes/private/main:note.md"], + false + ), + "legitimate descendant content" + ); +} + +#[test] +fn carrier_recovery_refuses_history_with_a_merge_hiding_a_legitimate_state() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + let source = temporary.path().join("source"); + let returning = temporary.path().join("returning"); + init_source(&source, "legitimate content"); + init_source(&returning, "returning local content"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + let repository = EncryptedRepository::new( + GitStorage::open(carrier.to_str().unwrap()).unwrap(), + owner.clone(), + ); + repository.initialize().unwrap(); + repository.pin_admin_state(&admin_pin).unwrap(); + let (floor_id, _) = repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + let floor_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let floor_bytes = floor_storage + .get_object(ObjectKind::Manifest, &floor_id) + .unwrap(); + let floor_header = peek_manifest_header(&floor_bytes).unwrap(); + let floor_key = unwrap_generation_key(&floor_header, &reader).unwrap(); + let floor_opened = open_manifest(&floor_bytes, &floor_key).unwrap(); + let floor_policy = PolicyState::parse( + floor_opened + .manifest() + .introduced_policy + .as_deref() + .unwrap(), + ) + .unwrap(); + repository.fetch_into(&returning, "private").unwrap(); + repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + + // Put a replay of the floor on the first-parent path and the legitimate + // tip on the second parent. The merge still fast-forwards the carrier. + let attacker = temporary.path().join("carrier-edit"); + git( + temporary.path(), + &[ + "clone", + "-q", + "-b", + "git-remote-e2ee", + carrier.to_str().unwrap(), + attacker.to_str().unwrap(), + ], + false, + ); + git(&attacker, &["config", "user.name", "Carrier editor"], false); + git( + &attacker, + &["config", "user.email", "carrier-editor@example.invalid"], + false, + ); + git( + &attacker, + &["checkout", "-q", "-b", "side", "HEAD~1"], + false, + ); + git( + &attacker, + &["commit", "-q", "--allow-empty", "-m", "replay floor"], + false, + ); + git( + &attacker, + &[ + "merge", + "-q", + "--no-ff", + "-s", + "ours", + "-m", + "hide", + "git-remote-e2ee", + ], + false, + ); + git( + &attacker, + &["push", "-q", "origin", "side:git-remote-e2ee"], + false, + ); + + let junk_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let junk_id = create_reader_signed_junk(&junk_storage, &reader, &floor_policy, &floor_id); + junk_storage + .compare_and_swap_head(Some(&floor_id), &junk_id) + .unwrap(); + + let recoverer = + EncryptedRepository::new(GitStorage::open(carrier.to_str().unwrap()).unwrap(), owner); + let error = recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("merge commit"), "{error:#}"); + let after = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + assert_eq!( + after.read_head().unwrap().as_deref(), + Some(junk_id.as_str()) + ); +} + +#[test] +fn carrier_recovery_refuses_conflicting_authenticated_descendants() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + let returning = temporary.path().join("returning"); + init_source(&returning, "local state"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + let repository = EncryptedRepository::new( + GitStorage::open(carrier.to_str().unwrap()).unwrap(), + owner.clone(), + ); + repository.initialize().unwrap(); + repository.pin_admin_state(&admin_pin).unwrap(); + let (floor_id, _) = repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + repository.fetch_into(&returning, "private").unwrap(); + + let policy_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let policy_encrypted = policy_storage + .get_object(ObjectKind::Manifest, &floor_id) + .unwrap(); + let policy_header = peek_manifest_header(&policy_encrypted).unwrap(); + let policy_key = unwrap_generation_key(&policy_header, &owner).unwrap(); + let policy_manifest = open_manifest(&policy_encrypted, &policy_key).unwrap(); + let policy = PolicyState::parse( + policy_manifest + .manifest() + .introduced_policy + .as_deref() + .unwrap(), + ) + .unwrap(); + + let first_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let first_fork = create_writer_successor( + &first_storage, + &owner, + &policy, + &floor_id, + BTreeMap::from([("refs/heads/main".to_owned(), "1".repeat(40))]), + ); + first_storage + .compare_and_swap_head(Some(&floor_id), &first_fork) + .unwrap(); + let second_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let second_fork = create_writer_successor( + &second_storage, + &owner, + &policy, + &floor_id, + BTreeMap::from([("refs/heads/main".to_owned(), "2".repeat(40))]), + ); + second_storage + .compare_and_swap_head(Some(&first_fork), &second_fork) + .unwrap(); + let junk_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let junk = create_reader_signed_junk(&junk_storage, &reader, &policy, &floor_id); + junk_storage + .compare_and_swap_head(Some(&second_fork), &junk) + .unwrap(); + + let recovery_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let report = EncryptedRepository::new(recovery_storage, owner) + .recover( + &returning, + "private", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert_eq!(report.classification, RecoveryClass::Invalid); + assert!(report.conflict); + assert!( + report + .blocked_reason + .as_deref() + .unwrap() + .contains("refusing to choose between forks") + ); + assert!( + report + .candidates + .iter() + .any(|candidate| candidate.manifest_id == first_fork) + ); + assert!( + report + .candidates + .iter() + .any(|candidate| candidate.manifest_id == second_fork) + ); + let current = GitStorage::open(carrier.to_str().unwrap()) + .unwrap() + .read_head() + .unwrap(); + assert_eq!(current.as_deref(), Some(junk.as_str())); +} + +#[test] +fn carrier_recovery_loses_cas_without_retrying_when_another_writer_publishes() { + let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); + let carrier = temporary.path().join("carrier.git"); + let returning = temporary.path().join("returning"); + init_source(&returning, "local state"); + git( + temporary.path(), + &["init", "--bare", "-q", carrier.to_str().unwrap()], + false, + ); + + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + let repository = EncryptedRepository::new( + GitStorage::open(carrier.to_str().unwrap()).unwrap(), + owner.clone(), + ); + repository.initialize().unwrap(); + repository.pin_admin_state(&admin_pin).unwrap(); + let (floor_id, _) = repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + repository.fetch_into(&returning, "private").unwrap(); + let policy_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let policy_encrypted = policy_storage + .get_object(ObjectKind::Manifest, &floor_id) + .unwrap(); + let policy_header = peek_manifest_header(&policy_encrypted).unwrap(); + let policy_key = unwrap_generation_key(&policy_header, &reader).unwrap(); + let policy_manifest = open_manifest(&policy_encrypted, &policy_key).unwrap(); + let policy = PolicyState::parse( + policy_manifest + .manifest() + .introduced_policy + .as_deref() + .unwrap(), + ) + .unwrap(); + + let bad_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let junk = create_reader_signed_junk(&bad_storage, &reader, &policy, &floor_id); + bad_storage + .compare_and_swap_head(Some(&floor_id), &junk) + .unwrap(); + let stale_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let stale_recoverer = EncryptedRepository::new(stale_storage, owner.clone()); + let offered = stale_recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: false, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert_eq!(offered.classification, RecoveryClass::Invalid); + assert_eq!(offered.default_base.as_deref(), Some(floor_id.as_str())); + + let competing_storage = GitStorage::open(carrier.to_str().unwrap()).unwrap(); + let winner = create_writer_successor( + &competing_storage, + &owner, + &policy, + &floor_id, + BTreeMap::new(), + ); + competing_storage + .compare_and_swap_head(Some(&junk), &winner) + .unwrap(); + + let error = stale_recoverer + .recover( + &returning, + "private", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("concurrently")); + let latest = + EncryptedRepository::new(GitStorage::open(carrier.to_str().unwrap()).unwrap(), owner); + assert_eq!(latest.current_manifest().unwrap().0, winner); +} + #[test] fn carrier_branch_coexists_with_unrelated_repository_history() { let temporary = tempfile::tempdir().unwrap(); + let _cache = isolated_cache(&temporary); let carrier = temporary.path().join("carrier.git"); let ordinary = temporary.path().join("ordinary"); init_source(&ordinary, "ordinary public content"); diff --git a/tests/cli_roles.rs b/tests/cli_roles.rs new file mode 100644 index 0000000..e19e80b --- /dev/null +++ b/tests/cli_roles.rs @@ -0,0 +1,106 @@ +use std::path::Path; +use std::process::{Command, Output}; + +use git_remote_e2ee::crypto::KeyFile; + +fn run_cli(args: &[String]) -> Output { + Command::new(env!("CARGO_BIN_EXE_git-e2ee")) + .args(args) + .output() + .unwrap() +} + +fn path_arg(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn add_device_args(storage: &Path, key: &Path, public: &Path) -> Vec { + vec![ + "device-add".to_owned(), + "--storage".to_owned(), + path_arg(storage), + "--key".to_owned(), + path_arg(key), + "--device".to_owned(), + path_arg(public), + ] +} + +#[test] +fn device_add_parses_roles_and_device_list_uses_role_names() { + let temporary = tempfile::tempdir().unwrap(); + let storage = temporary.path().join("storage"); + let owner_path = temporary.path().join("owner.key.json"); + let owner = KeyFile::generate(); + owner.write_new(&owner_path).unwrap(); + + let initialized = run_cli(&[ + "init".to_owned(), + "--storage".to_owned(), + path_arg(&storage), + "--key".to_owned(), + path_arg(&owner_path), + ]); + assert!( + initialized.status.success(), + "git-e2ee init: {}", + String::from_utf8_lossy(&initialized.stderr) + ); + + let roles = [ + ("default", None, "write"), + ("reader", Some("read"), "read"), + ("writer", Some("write"), "write"), + ("admin", Some("admin"), "admin"), + ]; + let mut device_ids = Vec::new(); + for (name, selected_role, expected_role) in roles { + let device_key = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let public = device_key.public_device().unwrap(); + let public_path = temporary.path().join(format!("{name}.public.json")); + public.write_new(&public_path).unwrap(); + let mut args = add_device_args(&storage, &owner_path, &public_path); + if let Some(role) = selected_role { + args.extend(["--role".to_owned(), role.to_owned()]); + } + let added = run_cli(&args); + assert!( + added.status.success(), + "git-e2ee device-add {name}: {}", + String::from_utf8_lossy(&added.stderr) + ); + device_ids.push((public.device_id, expected_role)); + } + + let mut legacy_args = add_device_args( + &storage, + &owner_path, + &temporary.path().join("default.public.json"), + ); + legacy_args.push("--admin".to_owned()); + let legacy = run_cli(&legacy_args); + assert!(!legacy.status.success()); + assert!(String::from_utf8_lossy(&legacy.stderr).contains("--admin")); + + let listed = run_cli(&[ + "device-list".to_owned(), + "--storage".to_owned(), + path_arg(&storage), + "--key".to_owned(), + path_arg(&owner_path), + ]); + assert!( + listed.status.success(), + "git-e2ee device-list: {}", + String::from_utf8_lossy(&listed.stderr) + ); + let output = String::from_utf8(listed.stdout).unwrap(); + for (device_id, role) in device_ids { + assert!( + output.contains(&format!("{device_id} role={role} status=active")), + "missing role {role} for {device_id} in:\n{output}" + ); + } + let owner_id = owner.device_id().unwrap(); + assert!(output.contains(&format!("{owner_id} role=admin status=active"))); +} diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index 104e394..894d179 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -1,15 +1,92 @@ use std::fs; +use std::io::Read; use std::path::Path; use std::process::Command; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier}; +use std::thread; +use anyhow::Result; +use base64::Engine; use git_remote_e2ee::crypto::{KeyFile, object_id, random_key}; use git_remote_e2ee::manifest::{ Manifest, ManifestAuthorization, open_manifest, peek_manifest_header, seal_manifest, - unwrap_generation_key, wrap_predecessor_key, + unwrap_generation_key, verify_manifest, wrap_predecessor_key, }; -use git_remote_e2ee::policy::PolicyState; -use git_remote_e2ee::repository::EncryptedRepository; -use git_remote_e2ee::storage::FilesystemStorage; +use git_remote_e2ee::policy::{DeviceRoles, PolicyState}; +use git_remote_e2ee::repository::{EncryptedRepository, RecoveryClass, RecoveryOptions}; +use git_remote_e2ee::storage::{ + FilesystemStorage, ObjectKind, ObjectStage, RecoveryCommit, RecoveryHistory, Storage, +}; + +#[derive(Clone)] +struct BarrierStorage { + inner: FilesystemStorage, + armed: Arc, + reads: Arc, + barrier: Arc, +} + +#[derive(Clone)] +struct OverBudgetHistoryStorage { + inner: FilesystemStorage, +} + +impl Storage for OverBudgetHistoryStorage { + fn begin_object(&self, kind: ObjectKind) -> Result> { + self.inner.begin_object(kind) + } + + fn open_object(&self, kind: ObjectKind, id: &str) -> Result> { + self.inner.open_object(kind, id) + } + + fn read_head(&self) -> Result> { + self.inner.read_head() + } + + fn compare_and_swap_head(&self, expected: Option<&str>, next: &str) -> Result<()> { + self.inner.compare_and_swap_head(expected, next) + } + + fn recovery_history( + &self, + max_commits: usize, + _max_manifest_bytes: u64, + ) -> Result> { + Ok(Some(RecoveryHistory { + commits: (0..=max_commits) + .map(|index| RecoveryCommit { + commit_id: format!("commit-{index}"), + head_id: self.inner.read_head().unwrap(), + }) + .collect(), + manifests: Default::default(), + })) + } +} + +impl Storage for BarrierStorage { + fn begin_object(&self, kind: ObjectKind) -> Result> { + self.inner.begin_object(kind) + } + + fn open_object(&self, kind: ObjectKind, id: &str) -> Result> { + self.inner.open_object(kind, id) + } + + fn read_head(&self) -> Result> { + let head = self.inner.read_head()?; + if self.armed.load(Ordering::SeqCst) && self.reads.fetch_add(1, Ordering::SeqCst) < 2 { + self.barrier.wait(); + } + Ok(head) + } + + fn compare_and_swap_head(&self, expected: Option<&str>, next: &str) -> Result<()> { + self.inner.compare_and_swap_head(expected, next) + } +} fn git(repo: &Path, args: &[&str]) -> String { let output = Command::new("git") @@ -44,6 +121,14 @@ fn commit(repo: &Path, contents: &str, message: &str) -> String { git(repo, &["rev-parse", "HEAD"]) } +fn client_state(repo: &Path, remote_name: &str) -> serde_json::Value { + let path = repo + .join(".git/git-remote-e2ee") + .join(remote_name) + .join("state.json"); + serde_json::from_slice(&fs::read(path).unwrap()).unwrap() +} + fn copy_tree(source: &Path, destination: &Path) { fs::create_dir_all(destination).unwrap(); for entry in fs::read_dir(source).unwrap() { @@ -102,6 +187,102 @@ fn pushes_incrementally_and_fetches_into_another_repository() { ); } +#[test] +fn missing_pack_observation_does_not_advance_the_floor() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let destination = temporary.path().join("destination"); + let remote_path = temporary.path().join("remote"); + initialize_git(&source); + initialize_git(&destination); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote_path), key); + repository.initialize().unwrap(); + commit(&source, "first\n", "first"); + repository + .push_update_for_remote( + &source, + "encrypted", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + repository.fetch_into(&destination, "encrypted").unwrap(); + assert_eq!(client_state(&destination, "encrypted")["generation"], 1); + assert_eq!( + client_state(&destination, "encrypted")["imported_generation"], + 1 + ); + + commit(&source, "second\n", "second"); + repository + .push_update_for_remote( + &source, + "encrypted", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + let (_, manifest) = repository.current_manifest().unwrap(); + let pack_id = manifest.new_packs[0].id.clone(); + let pack_path = remote_path + .join("objects") + .join(&pack_id[..2]) + .join(&pack_id); + let saved_pack = fs::read(&pack_path).unwrap(); + fs::remove_file(&pack_path).unwrap(); + + let listed = repository + .observe_manifest(&destination, "encrypted") + .unwrap(); + assert_eq!(listed.generation, 2); + assert_eq!(client_state(&destination, "encrypted")["generation"], 1); + let error = repository + .fetch_into(&destination, "encrypted") + .unwrap_err() + .to_string(); + assert!(error.contains(&pack_id), "unexpected error: {error}"); + assert_eq!(client_state(&destination, "encrypted")["generation"], 1); + + write_stored_object(&remote_path, "objects", &pack_id, &saved_pack); + let fetched = repository.fetch_into(&destination, "encrypted").unwrap(); + assert_eq!(fetched.generation, 2); + let state = client_state(&destination, "encrypted"); + assert_eq!(state["generation"], 2); + assert_eq!(state["imported_generation"], 2); +} + +#[test] +fn successful_own_publication_advances_the_client_floor() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let remote_path = temporary.path().join("remote"); + initialize_git(&source); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote_path), key); + repository.initialize().unwrap(); + commit(&source, "published\n", "published"); + let head = repository + .push_update_for_remote( + &source, + "encrypted", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + + let state_path = source.join(".git/git-remote-e2ee/encrypted/state.json"); + let state: serde_json::Value = serde_json::from_slice(&fs::read(state_path).unwrap()).unwrap(); + assert_eq!(state["head_id"], head); + assert_eq!(state["generation"], 1); + assert_eq!(state["imported_generation"], serde_json::Value::Null); +} + #[test] fn returning_client_fetches_multiple_offline_generations_from_deltas() { let temporary = tempfile::tempdir().unwrap(); @@ -135,7 +316,7 @@ fn returning_client_fetches_multiple_offline_generations_from_deltas() { } #[test] -fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { +fn returning_client_rejects_signed_ref_advance_without_its_pack() { let temporary = tempfile::tempdir().unwrap(); let source = temporary.path().join("source"); let destination = temporary.path().join("destination"); @@ -149,6 +330,7 @@ fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { repository .push_ref(&source, "refs/heads/main", false) .unwrap(); + repository.fetch_into(&destination, "e2ee").unwrap(); let head = fs::read_to_string(remote.join("HEAD")) .unwrap() @@ -156,11 +338,12 @@ fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { .to_owned(); let manifest_bytes = read_stored_object(&remote, "manifests", &head); let header = peek_manifest_header(&manifest_bytes).unwrap(); - let policy_bytes = read_stored_object(&remote, "policies", &header.policy_id); - let policy = PolicyState::parse(&policy_bytes).unwrap(); + let (policy, _) = PolicyState::genesis(&key).unwrap(); policy.validate_genesis(&key.repository_root).unwrap(); let current_key = unwrap_generation_key(&header, &key).unwrap(); - let current = open_manifest(&manifest_bytes, &policy, None, ¤t_key).unwrap(); + let current = open_manifest(&manifest_bytes, ¤t_key).unwrap(); + verify_manifest(¤t, &policy, None, ¤t_key).unwrap(); + let current = current.manifest().clone(); let next_key = random_key(); let generation = current.generation + 1; @@ -187,6 +370,7 @@ fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { ) .unwrap(), ), + introduced_policy: None, }; let malicious_bytes = seal_manifest( &key, @@ -208,6 +392,164 @@ fn signed_ref_advance_without_its_pack_is_rejected_by_connectivity() { error.contains("does not resolve") || error.contains("missing Git objects"), "unexpected connectivity error: {error}" ); + assert_eq!(client_state(&destination, "e2ee")["generation"], 1); + assert_eq!(client_state(&destination, "e2ee")["imported_generation"], 1); +} + +#[test] +fn missing_verified_frontier_tip_fails_closed() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let destination = temporary.path().join("destination"); + let remote = temporary.path().join("remote"); + initialize_git(&source); + initialize_git(&destination); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote), key); + repository.initialize().unwrap(); + commit(&source, "valid\n", "valid"); + repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + repository.fetch_into(&destination, "e2ee").unwrap(); + + let state_path = destination.join(".git/git-remote-e2ee/e2ee/state.json"); + let mut state: serde_json::Value = + serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + state["verified_refs"]["refs/heads/main"] = serde_json::Value::String("1".repeat(40)); + fs::write(&state_path, serde_json::to_vec_pretty(&state).unwrap()).unwrap(); + + let error = repository + .fetch_into(&destination, "e2ee") + .unwrap_err() + .to_string(); + assert!(error.contains("verified ref refs/heads/main is missing locally")); +} + +#[test] +fn local_publication_advances_verified_frontier_before_observation() { + const CHILD_ENV: &str = "E2EE_TEST_FRONTIER_TRACE_CHILD"; + if std::env::var_os(CHILD_ENV).is_none() { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "local_publication_advances_verified_frontier_before_observation", + "--nocapture", + ]) + .env(CHILD_ENV, "1") + .env("GIT_REMOTE_E2EE_TRACE", "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "frontier trace child failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let trace = String::from_utf8_lossy(&output.stderr); + let counters: Vec<_> = trace + .lines() + .filter(|line| line.contains("count=git_ref_connectivity_walk_objects")) + .collect(); + assert_eq!(counters.len(), 1, "expected one connectivity walk: {trace}"); + assert!( + counters[0].ends_with("value=0"), + "the observation re-walked published history: {}", + counters[0] + ); + return; + } + + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let remote = temporary.path().join("remote"); + initialize_git(&source); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote), key); + repository.initialize().unwrap(); + + let first = commit(&source, "first\n", "first"); + repository + .push_update_for_remote( + &source, + "origin", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + assert_eq!( + client_state(&source, "origin")["verified_refs"]["refs/heads/main"], + first + ); + + let observed = repository.observe_manifest(&source, "origin").unwrap(); + assert_eq!(observed.refs.get("refs/heads/main"), Some(&first)); + + let second = commit(&source, "second\n", "second"); + repository + .push_update_for_remote( + &source, + "origin", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + assert_eq!( + client_state(&source, "origin")["verified_refs"]["refs/heads/main"], + second + ); +} + +#[test] +fn shallow_publication_does_not_advance_verified_frontier() { + let temporary = tempfile::tempdir().unwrap(); + let full_source = temporary.path().join("full-source"); + let shallow_source = temporary.path().join("shallow-source"); + let remote = temporary.path().join("remote"); + initialize_git(&full_source); + commit(&full_source, "first\n", "first"); + commit(&full_source, "second\n", "second"); + + let source_url = format!("file://{}", full_source.display()); + let clone = Command::new("git") + .args(["clone", "--quiet", "--depth=1", "--no-tags"]) + .arg(&source_url) + .arg(&shallow_source) + .output() + .unwrap(); + assert!( + clone.status.success(), + "git clone: {}", + String::from_utf8_lossy(&clone.stderr) + ); + assert_eq!( + git(&shallow_source, &["rev-parse", "--is-shallow-repository"]), + "true" + ); + + let key = KeyFile::generate(); + let repository = EncryptedRepository::new(FilesystemStorage::new(&remote), key); + repository.initialize().unwrap(); + repository + .push_update_for_remote( + &shallow_source, + "origin", + "refs/heads/main", + "refs/heads/main", + false, + ) + .unwrap(); + + assert!( + client_state(&shallow_source, "origin")["verified_refs"] + .as_object() + .unwrap() + .is_empty() + ); } fn read_stored_object(root: &Path, kind: &str, id: &str) -> Vec { @@ -408,23 +750,446 @@ fn rejects_same_generation_manifest_fork_after_fetch_pins_history() { let fork = EncryptedRepository::new(FilesystemStorage::new(&fork_path), key); fork.initialize().unwrap(); let fork_head = fork.push_ref(&source, "refs/heads/main", false).unwrap(); - let fork_manifest = fork_path - .join("manifests") - .join(&fork_head[..2]) - .join(&fork_head); - let remote_manifest = remote_path - .join("manifests") - .join(&fork_head[..2]) - .join(&fork_head); - fs::create_dir_all(remote_manifest.parent().unwrap()).unwrap(); - fs::copy(fork_manifest, remote_manifest).unwrap(); - copy_tree(&fork_path.join("policies"), &remote_path.join("policies")); + copy_tree(&fork_path.join("manifests"), &remote_path.join("manifests")); fs::write(remote_path.join("HEAD"), format!("{fork_head}\n")).unwrap(); let error = encrypted.fetch_into(&destination, "encrypted").unwrap_err(); assert!(error.to_string().contains("history forked")); } +#[test] +fn recover_replaces_reader_signed_junk_from_the_directory_floor() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let returning = temporary.path().join("returning"); + let fresh = temporary.path().join("fresh"); + let storage_path = temporary.path().join("storage"); + initialize_git(&source); + initialize_git(&returning); + initialize_git(&fresh); + + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + let storage = FilesystemStorage::new(&storage_path); + let repository = EncryptedRepository::new(storage.clone(), owner.clone()); + repository.initialize().unwrap(); + repository.pin_admin_state(&admin_pin).unwrap(); + commit(&source, "recovery keeps this content\n", "base"); + repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + repository.fetch_into(&returning, "encrypted").unwrap(); + let (floor_id, floor_manifest) = repository.current_manifest().unwrap(); + let floor_generation = floor_manifest.generation; + let floor_state = client_state(&returning, "encrypted"); + assert_eq!(floor_state["head_id"], floor_id); + + let encrypted = storage.get_object(ObjectKind::Manifest, &floor_id).unwrap(); + let header = peek_manifest_header(&encrypted).unwrap(); + let floor_key = unwrap_generation_key(&header, &reader).unwrap(); + let opened = open_manifest(&encrypted, &floor_key).unwrap(); + let policy = + PolicyState::parse(opened.manifest().introduced_policy.as_deref().unwrap()).unwrap(); + let recovery_key = random_key(); + let junk = Manifest { + format_version: floor_manifest.format_version, + repository_root: floor_manifest.repository_root.clone(), + generation: floor_generation + 1, + previous: Some(floor_id.clone()), + policy_id: floor_manifest.policy_id.clone(), + policy_generation: floor_manifest.policy_generation, + authorization: ManifestAuthorization::Writer, + total_pack_count: floor_manifest.total_pack_count, + refs: floor_manifest.refs.clone(), + new_packs: Vec::new(), + predecessor_key_wrap: Some( + wrap_predecessor_key( + &recovery_key, + &floor_key, + &floor_manifest.repository_root, + floor_generation + 1, + &floor_id, + ) + .unwrap(), + ), + introduced_policy: None, + }; + let junk_bytes = seal_manifest( + &reader, + &policy, + &recovery_key, + ManifestAuthorization::Writer, + junk, + ) + .unwrap(); + let junk_id = object_id(&junk_bytes); + storage + .put_object_if_absent(ObjectKind::Manifest, &junk_id, &junk_bytes) + .unwrap(); + storage + .compare_and_swap_head(Some(&floor_id), &junk_id) + .unwrap(); + + let recoverer = EncryptedRepository::new(storage.clone(), owner.clone()); + let blocked = recoverer + .recover( + &returning, + "encrypted", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert_eq!(blocked.classification, RecoveryClass::Invalid); + assert_eq!(blocked.default_base.as_deref(), Some(floor_id.as_str())); + assert!( + blocked + .warning + .as_deref() + .unwrap() + .starts_with("Updates after this floor") + ); + assert!( + blocked + .blocked_reason + .as_deref() + .unwrap() + .contains("--accept-stale-floor") + ); + assert_eq!(client_state(&returning, "encrypted")["head_id"], floor_id); + + let recovered = recoverer + .recover( + &returning, + "encrypted", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: true, + }, + ) + .unwrap(); + let recovered_id = recovered.published_manifest.unwrap(); + let (current_id, current) = recoverer.current_manifest().unwrap(); + assert_eq!(current_id, recovered_id); + assert_eq!(current.previous.as_deref(), Some(floor_id.as_str())); + assert_eq!(current.generation, floor_generation + 1); + assert_ne!(current_id, junk_id); + assert_eq!(client_state(&returning, "encrypted")["head_id"], current_id); + let rejected = + fs::read(returning.join(".git/git-remote-e2ee/encrypted/rejected-heads.json")).unwrap(); + assert!(String::from_utf8_lossy(&rejected).contains(&junk_id)); + + recoverer.fetch_into(&returning, "encrypted").unwrap(); + recoverer.fetch_into(&fresh, "encrypted").unwrap(); + assert_eq!( + git(&fresh, &["show", "refs/remotes/encrypted/main:note.md"]), + "recovery keeps this content" + ); + assert_eq!(client_state(&returning, "encrypted")["head_id"], current_id); + assert_eq!(client_state(&fresh, "encrypted")["head_id"], current_id); +} + +#[test] +fn recover_refuses_a_revoked_devices_unverifiable_head() { + let temporary = tempfile::tempdir().unwrap(); + let reader_repo = temporary.path().join("reader"); + let storage_path = temporary.path().join("storage"); + initialize_git(&reader_repo); + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + let storage = FilesystemStorage::new(&storage_path); + let owner_repository = EncryptedRepository::new(storage.clone(), owner); + owner_repository.initialize().unwrap(); + owner_repository.pin_admin_state(&admin_pin).unwrap(); + owner_repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + owner_repository + .fetch_into(&reader_repo, "encrypted") + .unwrap(); + let old_floor = client_state(&reader_repo, "encrypted")["head_id"] + .as_str() + .unwrap() + .to_owned(); + owner_repository + .revoke_device(&reader.device_id().unwrap(), &admin_pin) + .unwrap(); + let revoked_head = storage.read_head().unwrap().unwrap(); + + let reader_repository = EncryptedRepository::new(storage.clone(), reader); + let report = reader_repository + .recover( + &reader_repo, + "encrypted", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: true, + }, + ) + .unwrap(); + assert_eq!(report.classification, RecoveryClass::Unverifiable); + assert!(report.blocked_reason.unwrap().contains("forbidden")); + assert_eq!( + storage.read_head().unwrap().as_deref(), + Some(revoked_head.as_str()) + ); + assert_eq!( + client_state(&reader_repo, "encrypted")["head_id"], + old_floor + ); + assert!( + !reader_repo + .join(".git/git-remote-e2ee/encrypted/rejected-heads.json") + .exists() + ); +} + +#[test] +fn recover_refuses_unknown_formats_and_missing_floor_objects() { + let temporary = tempfile::tempdir().unwrap(); + + let unknown_repo = temporary.path().join("unknown-repo"); + let unknown_storage_path = temporary.path().join("unknown-storage"); + initialize_git(&unknown_repo); + let key = KeyFile::generate(); + let unknown_storage = FilesystemStorage::new(&unknown_storage_path); + let unknown_repository = EncryptedRepository::new(unknown_storage.clone(), key.clone()); + unknown_repository.initialize().unwrap(); + unknown_repository + .fetch_into(&unknown_repo, "encrypted") + .unwrap(); + let floor_id = unknown_storage.read_head().unwrap().unwrap(); + let header = serde_json::json!({ + "format_version": 99, + "repository_root": key.repository_root, + "generation": 1, + "previous": floor_id, + "key_commitment": "0".repeat(64), + "generation_key_envelopes": [], + "body_digest": "0".repeat(64), + "sealed_header_digest": "0".repeat(64) + }); + let header_bytes = serde_json::to_vec(&header).unwrap(); + let unsupported_bytes = serde_json::to_vec(&serde_json::json!({ + "header": base64::engine::general_purpose::STANDARD.encode(header_bytes), + "sealed_header_ciphertext": "", + "body_ciphertext": "" + })) + .unwrap(); + let unsupported_id = object_id(&unsupported_bytes); + unknown_storage + .put_object_if_absent(ObjectKind::Manifest, &unsupported_id, &unsupported_bytes) + .unwrap(); + unknown_storage + .compare_and_swap_head(Some(&floor_id), &unsupported_id) + .unwrap(); + let unsupported = EncryptedRepository::new(unknown_storage.clone(), key) + .recover( + &unknown_repo, + "encrypted", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: true, + }, + ) + .unwrap(); + assert_eq!(unsupported.classification, RecoveryClass::Unsupported); + assert_eq!( + unknown_storage.read_head().unwrap().as_deref(), + Some(unsupported_id.as_str()) + ); + + let missing_repo = temporary.path().join("missing-repo"); + let missing_storage_path = temporary.path().join("missing-storage"); + initialize_git(&missing_repo); + let missing_key = KeyFile::generate(); + let missing_storage = FilesystemStorage::new(&missing_storage_path); + let missing_repository = EncryptedRepository::new(missing_storage.clone(), missing_key.clone()); + missing_repository.initialize().unwrap(); + let source = temporary.path().join("missing-source"); + initialize_git(&source); + commit(&source, "required floor object\n", "required"); + missing_repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + missing_repository + .fetch_into(&missing_repo, "encrypted") + .unwrap(); + let floor_id = missing_storage.read_head().unwrap().unwrap(); + let floor_manifest = missing_repository.current_manifest().unwrap().1; + let missing_pack = &floor_manifest.new_packs[0].id; + fs::remove_file( + missing_storage_path + .join("objects") + .join(&missing_pack[..2]) + .join(missing_pack), + ) + .unwrap(); + let unavailable = missing_repository + .recover( + &missing_repo, + "encrypted", + RecoveryOptions { + publish: false, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap(); + assert_eq!(unavailable.classification, RecoveryClass::Unavailable); + let invalid_bytes = b"invalid successor"; + let invalid_id = object_id(invalid_bytes); + missing_storage + .put_object_if_absent(ObjectKind::Manifest, &invalid_id, invalid_bytes) + .unwrap(); + missing_storage + .compare_and_swap_head(Some(&floor_id), &invalid_id) + .unwrap(); + let refused = EncryptedRepository::new(missing_storage.clone(), missing_key) + .recover( + &missing_repo, + "encrypted", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: true, + }, + ) + .unwrap(); + assert_eq!(refused.classification, RecoveryClass::Invalid); + assert!( + refused + .blocked_reason + .as_deref() + .unwrap() + .contains("ciphertext is unavailable") + ); + assert_eq!( + missing_storage.read_head().unwrap().as_deref(), + Some(invalid_id.as_str()) + ); + assert_eq!( + client_state(&missing_repo, "encrypted")["head_id"], + floor_id + ); +} + +#[test] +fn recover_never_overrides_authenticated_rollback() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let returning = temporary.path().join("returning"); + let storage_path = temporary.path().join("storage"); + initialize_git(&source); + initialize_git(&returning); + let key = KeyFile::generate(); + let storage = FilesystemStorage::new(&storage_path); + let repository = EncryptedRepository::new(storage.clone(), key); + repository.initialize().unwrap(); + commit(&source, "one\n", "one"); + let first = repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + commit(&source, "two\n", "two"); + repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + repository.fetch_into(&returning, "encrypted").unwrap(); + storage + .compare_and_swap_head(Some(&repository.current_manifest().unwrap().0), &first) + .unwrap(); + + let report = repository + .recover( + &returning, + "encrypted", + RecoveryOptions { + publish: true, + base: None, + discard_newer: true, + accept_stale_floor: true, + }, + ) + .unwrap(); + assert_eq!(report.classification, RecoveryClass::Discontinuous); + assert!(report.blocked_reason.unwrap().contains("forbidden")); + assert_eq!( + storage.read_head().unwrap().as_deref(), + Some(first.as_str()) + ); +} + +#[test] +fn recovery_discovery_budget_exhaustion_fails_closed() { + let temporary = tempfile::tempdir().unwrap(); + let repo = temporary.path().join("repo"); + let storage_path = temporary.path().join("storage"); + initialize_git(&repo); + let key = KeyFile::generate(); + let inner = FilesystemStorage::new(&storage_path); + let repository = EncryptedRepository::new(inner.clone(), key.clone()); + repository.initialize().unwrap(); + repository.fetch_into(&repo, "encrypted").unwrap(); + let floor = inner.read_head().unwrap().unwrap(); + let invalid_bytes = b"malformed successor"; + let invalid_id = object_id(invalid_bytes); + inner + .put_object_if_absent(ObjectKind::Manifest, &invalid_id, invalid_bytes) + .unwrap(); + inner + .compare_and_swap_head(Some(&floor), &invalid_id) + .unwrap(); + + let recovery = EncryptedRepository::new( + OverBudgetHistoryStorage { + inner: inner.clone(), + }, + key, + ); + let error = recovery + .recover( + &repo, + "encrypted", + RecoveryOptions { + publish: false, + base: None, + discard_newer: false, + accept_stale_floor: false, + }, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("outer-commit budget exhausted")); + assert_eq!( + inner.read_head().unwrap().as_deref(), + Some(invalid_id.as_str()) + ); + assert_eq!(client_state(&repo, "encrypted")["head_id"], floor); +} + #[test] fn writer_can_add_branch_without_having_other_remote_branch_objects() { let temporary = tempfile::tempdir().unwrap(); @@ -489,7 +1254,67 @@ fn writer_can_add_branch_without_having_other_remote_branch_objects() { } #[test] -fn rejects_tag_destinations_consistently() { +fn deletion_and_tag_publication_race_with_exactly_one_cas_winner() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let remote = temporary.path().join("remote"); + initialize_git(&source); + let key = KeyFile::generate(); + let seed_repository = EncryptedRepository::new(FilesystemStorage::new(&remote), key.clone()); + seed_repository.initialize().unwrap(); + let main = commit(&source, "main\n", "main"); + seed_repository + .push_ref(&source, "refs/heads/main", false) + .unwrap(); + git(&source, &["branch", "doomed", &main]); + seed_repository + .push_ref(&source, "refs/heads/doomed", false) + .unwrap(); + git(&source, &["tag", "race-tag", &main]); + let before = seed_repository.verify().unwrap(); + + let storage = BarrierStorage { + inner: FilesystemStorage::new(&remote), + armed: Arc::new(AtomicBool::new(true)), + reads: Arc::new(AtomicUsize::new(0)), + barrier: Arc::new(Barrier::new(2)), + }; + let deleter = EncryptedRepository::new(storage.clone(), key.clone()); + let tagger = EncryptedRepository::new(storage, key); + let deletion_repo = source.clone(); + let tag_repo = source.clone(); + let deletion = thread::spawn(move || deleter.delete_ref(&deletion_repo, "refs/heads/doomed")); + let tag_push = thread::spawn(move || { + tagger.push_update(&tag_repo, "refs/tags/race-tag", "refs/tags/race-tag", false) + }); + let deletion = deletion.join().unwrap(); + let tag_push = tag_push.join().unwrap(); + assert_ne!(deletion.is_ok(), tag_push.is_ok()); + let loser = if deletion.is_err() { + deletion + } else { + tag_push + }; + assert!( + loser + .unwrap_err() + .to_string() + .contains("head changed concurrently") + ); + + let after = seed_repository.verify().unwrap(); + assert_eq!(after.generation, before.generation + 1); + let deletion_won = !after.refs.contains_key("refs/heads/doomed"); + let tag_push_won = after.refs.contains_key("refs/tags/race-tag"); + assert_ne!(deletion_won, tag_push_won); + assert_eq!( + after.total_pack_count, + before.total_pack_count + u64::from(tag_push_won) + ); +} + +#[test] +fn rejects_unsupported_ref_namespaces_consistently() { let temporary = tempfile::tempdir().unwrap(); let source = temporary.path().join("source"); let remote_path = temporary.path().join("remote"); @@ -499,12 +1324,10 @@ fn rejects_tag_destinations_consistently() { let encrypted = EncryptedRepository::new(FilesystemStorage::new(&remote_path), key); encrypted.initialize().unwrap(); commit(&source, "tagged\n", "tagged"); - git(&source, &["tag", "v1"]); - let error = encrypted - .push_update(&source, "refs/tags/v1", "refs/tags/v1", false) + .push_update(&source, "refs/heads/main", "refs/notes/review", false) .unwrap_err(); - assert!(error.to_string().contains("only refs/heads")); + assert!(error.to_string().contains("unsupported ref namespace")); assert_eq!(encrypted.verify().unwrap().generation, 0); } @@ -544,7 +1367,7 @@ fn init_with_relative_storage_creates_the_store_in_the_child_cwd() { ); assert!(work.join("relative-store/objects").is_dir()); assert!(work.join("relative-store/manifests").is_dir()); - assert!(work.join("relative-store/policies").is_dir()); + assert!(!work.join("relative-store/policies").exists()); assert!(!root.path().join("relative-store").exists()); } diff --git a/tests/git_compatibility.rs b/tests/git_compatibility.rs index 9b1d120..0f1da07 100644 --- a/tests/git_compatibility.rs +++ b/tests/git_compatibility.rs @@ -185,6 +185,234 @@ fn upstream_style_push_refspecs_and_force_work() { ); } +#[test] +fn lightweight_annotated_and_followed_tags_round_trip_through_helper() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let before_tags = temporary.path().join("before-tags"); + initialize_git(&source); + let (storage, key_path, key) = setup_remote(temporary.path()); + configure_remote(&source, &storage, &key_path); + let first = commit_file(&source, "first", "first"); + git(&source, &["push", "private", "main"], true); + clone_remote(&before_tags, &storage, &key_path); + + git(&source, &["tag", "v1.0", &first], false); + git(&source, &["push", "private", "v1.0"], true); + git( + &source, + &["tag", "-a", "v1.1", "-m", "release one", &first], + false, + ); + fs::write(source.join("tag-target.bin"), b"tagged blob object\n").unwrap(); + let blob = git(&source, &["hash-object", "-w", "tag-target.bin"], false); + git(&source, &["add", "tag-target.bin"], false); + let tree = git(&source, &["write-tree"], false); + git( + &source, + &["tag", "-a", "v-blob", "-m", "blob target", &blob], + false, + ); + git( + &source, + &["tag", "-a", "v-tree", "-m", "tree target", &tree], + false, + ); + git(&source, &["push", "--tags", "private"], true); + + git(&before_tags, &["fetch", "--tags", "origin"], true); + assert_eq!( + git(&before_tags, &["rev-parse", "refs/tags/v1.0"], false), + first + ); + assert_eq!( + git(&before_tags, &["cat-file", "-t", "refs/tags/v1.1"], false), + "tag" + ); + assert_eq!( + git(&before_tags, &["rev-parse", "refs/tags/v1.1^{}"], false), + first + ); + assert_eq!( + git(&before_tags, &["cat-file", "-t", "refs/tags/v-blob"], false), + "tag" + ); + assert_eq!( + git(&before_tags, &["rev-parse", "refs/tags/v-blob^{}"], false), + blob + ); + git(&before_tags, &["cat-file", "-e", &blob], false); + assert_eq!( + git(&before_tags, &["rev-parse", "refs/tags/v-tree^{}"], false), + tree + ); + assert_eq!(git(&before_tags, &["cat-file", "-t", &tree], false), "tree"); + + let second = commit_file(&source, "second", "second"); + git( + &source, + &["tag", "-a", "v2.0", "-m", "release two", &second], + false, + ); + git(&source, &["push", "--follow-tags", "private", "main"], true); + + let repository = EncryptedRepository::new(FilesystemStorage::new(&storage), key); + let manifest = repository.verify().unwrap(); + assert_eq!(manifest.refs["refs/tags/v1.0"], first); + assert_eq!( + manifest.refs["refs/tags/v1.1"], + git(&source, &["rev-parse", "refs/tags/v1.1"], false) + ); + assert_eq!( + manifest.refs["refs/tags/v2.0"], + git(&source, &["rev-parse", "refs/tags/v2.0"], false) + ); + + let fresh = temporary.path().join("fresh-clone"); + clone_remote( + &fresh, + &storage, + &temporary.path().join("repository.key.json"), + ); + assert_eq!( + git(&fresh, &["cat-file", "-t", "refs/tags/v1.1"], false), + "tag" + ); + assert_eq!( + git(&fresh, &["cat-file", "-t", "refs/tags/v2.0"], false), + "tag" + ); + assert_eq!( + git(&fresh, &["rev-parse", "refs/tags/v2.0^{}"], false), + second + ); +} + +#[test] +fn moving_an_existing_tag_requires_force() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + initialize_git(&source); + let (storage, key_path, key) = setup_remote(temporary.path()); + configure_remote(&source, &storage, &key_path); + let first = commit_file(&source, "first", "first"); + git(&source, &["push", "private", "main"], true); + git(&source, &["tag", "v1.0", &first], false); + git(&source, &["push", "private", "v1.0"], true); + + let second = commit_file(&source, "second", "second"); + git(&source, &["push", "private", "main"], true); + git(&source, &["tag", "-f", "v1.0", &second], false); + let repository = EncryptedRepository::new(FilesystemStorage::new(&storage), key); + let before = repository.verify().unwrap(); + let rejected = git_output(&source, &["push", "private", "v1.0"], true); + assert!(!rejected.status.success()); + let message = format!( + "{}{}", + String::from_utf8_lossy(&rejected.stdout), + String::from_utf8_lossy(&rejected.stderr) + ); + assert!(message.contains("already exists"), "{message}"); + let after_rejection = repository.verify().unwrap(); + assert_eq!(after_rejection.generation, before.generation); + assert_eq!(after_rejection.refs["refs/tags/v1.0"], first); + + git(&source, &["push", "--force", "private", "v1.0"], true); + assert_eq!(repository.verify().unwrap().refs["refs/tags/v1.0"], second); +} + +#[test] +fn deletions_publish_without_packs_and_prune_on_returning_and_fresh_clones() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let returning = temporary.path().join("returning-clone"); + initialize_git(&source); + let (storage, key_path, key) = setup_remote(temporary.path()); + configure_remote(&source, &storage, &key_path); + let main = commit_file(&source, "main", "main"); + git(&source, &["push", "private", "main"], true); + git(&source, &["branch", "doomed", &main], false); + git(&source, &["push", "private", "doomed"], true); + git(&source, &["tag", "v-delete", &main], false); + git(&source, &["push", "private", "v-delete"], true); + clone_remote(&returning, &storage, &key_path); + + let repository = EncryptedRepository::new(FilesystemStorage::new(&storage), key); + let before = repository.verify().unwrap(); + let default_delete = git_output(&source, &["push", "--delete", "private", "main"], true); + assert!(!default_delete.status.success()); + let default_error = format!( + "{}{}", + String::from_utf8_lossy(&default_delete.stdout), + String::from_utf8_lossy(&default_delete.stderr) + ); + assert!( + default_error.contains("remote default branch"), + "{default_error}" + ); + assert_eq!(repository.verify().unwrap().generation, before.generation); + + git(&source, &["push", "--delete", "private", "doomed"], true); + let after_branch_delete = repository.verify().unwrap(); + assert!(!after_branch_delete.refs.contains_key("refs/heads/doomed")); + assert_eq!( + after_branch_delete.total_pack_count, + before.total_pack_count + ); + + git(&source, &["push", "private", ":refs/tags/v-delete"], true); + let after_tag_delete = repository.verify().unwrap(); + assert!(!after_tag_delete.refs.contains_key("refs/tags/v-delete")); + assert_eq!(after_tag_delete.total_pack_count, before.total_pack_count); + + git( + &returning, + &["fetch", "--prune", "--prune-tags", "origin"], + true, + ); + assert!( + git_output( + &returning, + &["show-ref", "--verify", "refs/remotes/origin/doomed"], + false + ) + .status + .code() + .is_some_and(|code| code != 0) + ); + assert!( + git_output( + &returning, + &["show-ref", "--verify", "refs/tags/v-delete"], + false + ) + .status + .code() + .is_some_and(|code| code != 0) + ); + + let fresh = temporary.path().join("fresh-clone"); + clone_remote(&fresh, &storage, &key_path); + assert!( + !git_output( + &fresh, + &["show-ref", "--verify", "refs/remotes/origin/doomed"], + false + ) + .status + .success() + ); + assert!( + !git_output( + &fresh, + &["show-ref", "--verify", "refs/tags/v-delete"], + false + ) + .status + .success() + ); +} + #[test] fn unsupported_upstream_operations_fail_without_publication() { let temporary = tempfile::tempdir().unwrap(); @@ -204,10 +432,19 @@ fn unsupported_upstream_operations_fail_without_publication() { assert_eq!(after_delete.generation, generation); assert_eq!(after_delete.refs["refs/heads/main"], head); - git(&source, &["tag", "v1"], false); - let tag = git_output(&source, &["push", "private", "v1"], true); - assert!(!tag.status.success()); - let after_tag = repository.verify().unwrap(); - assert_eq!(after_tag.generation, generation); - assert!(!after_tag.refs.contains_key("refs/tags/v1")); + let namespace = git_output( + &source, + &["push", "private", "HEAD:refs/notes/review"], + true, + ); + assert!(!namespace.status.success()); + let message = format!( + "{}{}", + String::from_utf8_lossy(&namespace.stdout), + String::from_utf8_lossy(&namespace.stderr) + ); + assert!(message.contains("unsupported ref namespace"), "{message}"); + let after_namespace = repository.verify().unwrap(); + assert_eq!(after_namespace.generation, generation); + assert!(!after_namespace.refs.contains_key("refs/notes/review")); } diff --git a/tests/multi_device.rs b/tests/multi_device.rs index fc1b370..c62fb1d 100644 --- a/tests/multi_device.rs +++ b/tests/multi_device.rs @@ -225,7 +225,7 @@ fn revoked_reader_cannot_reach_later_history_but_keeps_prior_snapshot() { } #[test] -fn membership_observation_pins_head_without_skipping_later_pack_import() { +fn connected_membership_observation_pins_and_rejects_replay() { let temp = tempfile::tempdir().unwrap(); let remote = temp.path().join("remote"); let source = temp.path().join("source"); @@ -246,6 +246,9 @@ fn membership_observation_pins_head_without_skipping_later_pack_import() { repository.fetch_into(&client, "e2ee").unwrap(); let before_membership = fs::read_to_string(remote.join("HEAD")).unwrap(); + // The membership change keeps every ref, so the client already holds a + // fully connected copy of the advertised state. Native Git would not send + // a fetch command here, so observation itself must raise the floor. repository .add_device( key_b.public_device().unwrap(), @@ -253,15 +256,19 @@ fn membership_observation_pins_head_without_skipping_later_pack_import() { &admin_pin, ) .unwrap(); - repository.observe_manifest(&client, "e2ee").unwrap(); + let observed = repository.observe_manifest(&client, "e2ee").unwrap(); + assert_eq!(observed.generation, 2); + let state_path = client.join(".git/git-remote-e2ee/e2ee/state.json"); + let state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + assert_eq!(state["generation"], 2); + assert_eq!(state["policy_generation"], 1); let after_membership = fs::read_to_string(remote.join("HEAD")).unwrap(); fs::write(remote.join("HEAD"), &before_membership).unwrap(); - let error = repository - .observe_manifest(&client, "e2ee") - .unwrap_err() - .to_string(); - assert!(error.contains("rolled back"), "unexpected error: {error}"); + let error = repository.observe_manifest(&client, "e2ee").unwrap_err(); + assert!(format!("{error:#}").contains("rolled back"), "{error:#}"); + let state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + assert_eq!(state["generation"], 2); fs::write(remote.join("HEAD"), after_membership).unwrap(); commit(&source, "two\n", "two"); @@ -341,6 +348,49 @@ fn non_admin_device_cannot_change_policy() { ); } +#[test] +fn read_only_device_cannot_add_or_revoke_devices() { + let temp = tempfile::tempdir().unwrap(); + let remote = temp.path().join("remote"); + let owner_key = KeyFile::generate(); + let reader_key = KeyFile::generate_for_repository(owner_key.repository_root.clone()).unwrap(); + let candidate_key = + KeyFile::generate_for_repository(owner_key.repository_root.clone()).unwrap(); + let admin_pin = temp.path().join("owner-admin-state.json"); + let owner = EncryptedRepository::new(FilesystemStorage::new(&remote), owner_key); + owner.initialize().unwrap(); + owner.pin_admin_state(&admin_pin).unwrap(); + owner + .add_device( + reader_key.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + + let reader = EncryptedRepository::new(FilesystemStorage::new(&remote), reader_key.clone()); + let before = snapshot_files(&remote); + assert!( + reader + .add_device( + candidate_key.public_device().unwrap(), + DeviceRoles::collaborator(), + &admin_pin, + ) + .unwrap_err() + .to_string() + .contains("not an active administrator") + ); + assert!( + reader + .revoke_device(&reader_key.device_id().unwrap(), &admin_pin) + .unwrap_err() + .to_string() + .contains("not an active administrator") + ); + assert_eq!(snapshot_files(&remote), before); +} + fn copy_directory(source: &Path, destination: &Path) { fs::create_dir_all(destination).unwrap(); for entry in fs::read_dir(source).unwrap() { diff --git a/tests/read_only_helper.rs b/tests/read_only_helper.rs new file mode 100644 index 0000000..77d4af0 --- /dev/null +++ b/tests/read_only_helper.rs @@ -0,0 +1,311 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::{Mutex, MutexGuard}; + +use git_remote_e2ee::crypto::KeyFile; +use git_remote_e2ee::policy::DeviceRoles; +use git_remote_e2ee::repository::{EncryptedRepository, RecoveryClass, RecoveryOptions}; +use git_remote_e2ee::storage::{FilesystemStorage, GitStorage, ObjectKind, Storage}; + +static CACHE_ENV_LOCK: Mutex<()> = Mutex::new(()); + +fn git_output(repo: &Path, args: &[&str], with_helper: bool) -> Output { + let mut command = Command::new("git"); + command.arg("-C").arg(repo).args(args); + if with_helper { + add_helper_to_path(&mut command); + } + command.output().unwrap() +} + +fn git(repo: &Path, args: &[&str], with_helper: bool) -> String { + let output = git_output(repo, args, with_helper); + assert!( + output.status.success(), + "git {:?}: stdout={} stderr={}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_owned() +} + +fn add_helper_to_path(command: &mut Command) { + let helper = Path::new(env!("CARGO_BIN_EXE_git-remote-e2ee")); + let mut paths = vec![helper.parent().unwrap().to_path_buf()]; + paths.extend(env::split_paths(&env::var_os("PATH").unwrap_or_default())); + command.env("PATH", env::join_paths(paths).unwrap()); +} + +fn initialize_git(repo: &Path) { + fs::create_dir_all(repo).unwrap(); + git(repo, &["init", "-q", "-b", "main"], false); + git(repo, &["config", "user.name", "Read Only Test"], false); + git( + repo, + &["config", "user.email", "read-only@example.invalid"], + false, + ); +} + +fn commit(repo: &Path, contents: &str, message: &str) -> String { + fs::write(repo.join("note.md"), contents).unwrap(); + git(repo, &["add", "note.md"], false); + git(repo, &["commit", "-q", "-m", message], false); + git(repo, &["rev-parse", "HEAD"], false) +} + +fn configure_remote(repo: &Path, name: &str, url: &str, key_path: &Path) { + git(repo, &["remote", "add", name, url], false); + git( + repo, + &[ + "config", + &format!("remote.{name}.e2ee-key"), + key_path.to_str().unwrap(), + ], + false, + ); +} + +fn snapshot_tree(root: &Path) -> Vec<(PathBuf, Vec)> { + fn visit(root: &Path, current: &Path, files: &mut Vec<(PathBuf, Vec)>) { + for entry in fs::read_dir(current).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + visit(root, &path, files); + } else { + files.push(( + path.strip_prefix(root).unwrap().to_path_buf(), + fs::read(path).unwrap(), + )); + } + } + } + + let mut files = Vec::new(); + visit(root, root, &mut files); + files.sort_by(|left, right| left.0.cmp(&right.0)); + files +} + +fn exercise_read_only_helper( + temporary: &Path, + storage: S, + remote_path: &Path, + remote_url: &str, +) { + fs::create_dir_all(temporary).unwrap(); + let owner_key_path = temporary.join("owner.key.json"); + let reader_key_path = temporary.join("reader.key.json"); + let owner_key = KeyFile::generate(); + let reader_key = KeyFile::generate_for_repository(owner_key.repository_root.clone()).unwrap(); + owner_key.write_new(&owner_key_path).unwrap(); + reader_key.write_new(&reader_key_path).unwrap(); + + let owner = EncryptedRepository::new(storage, owner_key); + owner.initialize().unwrap(); + let admin_pin = temporary.join("owner.admin-state.json"); + owner.pin_admin_state(&admin_pin).unwrap(); + owner + .add_device( + reader_key.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + + let source = temporary.join("writer"); + initialize_git(&source); + let first = commit(&source, "initial writer content\n", "initial"); + configure_remote(&source, "private", remote_url, &owner_key_path); + git(&source, &["push", "private", "main"], true); + + let clone = temporary.join("reader-clone"); + let mut clone_command = Command::new("git"); + clone_command + .args([ + "-c", + &format!("e2ee.key={}", reader_key_path.display()), + "clone", + "-q", + remote_url, + ]) + .arg(&clone); + add_helper_to_path(&mut clone_command); + let output = clone_command.output().unwrap(); + assert!( + output.status.success(), + "reader clone failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(git(&clone, &["rev-parse", "HEAD"], false), first); + assert_eq!( + fs::read_to_string(clone.join("note.md")).unwrap(), + "initial writer content\n" + ); + + let second = commit(&source, "fetched writer content\n", "second"); + git(&source, &["push", "private", "main"], true); + git(&clone, &["fetch", "origin"], true); + assert_eq!( + git(&clone, &["rev-parse", "refs/remotes/origin/main"], false), + second + ); + + let third = commit(&source, "pulled writer content\n", "third"); + git(&source, &["push", "private", "main"], true); + git(&clone, &["pull", "--ff-only"], true); + assert_eq!(git(&clone, &["rev-parse", "HEAD"], false), third); + assert_eq!( + fs::read_to_string(clone.join("note.md")).unwrap(), + "pulled writer content\n" + ); + + // A fresh clone has no identity; CI runners have no global Git identity. + git(&clone, &["config", "user.name", "Read Only Test"], false); + git( + &clone, + &["config", "user.email", "read-only@example.invalid"], + false, + ); + commit(&clone, "local reader change\n", "reader change"); + let expected_error = + "this device is read-only for origin; ask an administrator for the write role"; + for args in [ + vec!["push", "origin", "HEAD:refs/heads/main"], + vec!["push", "--dry-run", "origin", "HEAD:refs/heads/main"], + vec!["push", "--force", "origin", "HEAD:refs/heads/main"], + ] { + let before = snapshot_tree(remote_path); + let output = git_output(&clone, &args, true); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "push unexpectedly succeeded: {stderr}" + ); + assert!( + stderr.contains(expected_error), + "push error did not explain the read-only role: {stderr}" + ); + assert_eq!( + snapshot_tree(remote_path), + before, + "rejected push changed storage" + ); + } + + let before = snapshot_tree(remote_path); + let output = git_output(&clone, &["push", "--delete", "origin", "main"], true); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "read-only delete unexpectedly succeeded" + ); + assert!( + stderr.contains(expected_error), + "delete error did not explain the read-only role: {stderr}" + ); + assert_eq!( + snapshot_tree(remote_path), + before, + "rejected delete changed storage" + ); +} + +#[test] +fn read_only_devices_use_both_storage_backends_without_publishing() { + let temporary = tempfile::tempdir().unwrap(); + let filesystem_remote = temporary.path().join("filesystem-remote"); + let filesystem_url = format!("e2ee::{}", filesystem_remote.display()); + exercise_read_only_helper( + temporary.path(), + FilesystemStorage::new(&filesystem_remote), + &filesystem_remote, + &filesystem_url, + ); + + let _cache_lock: MutexGuard<'static, ()> = CACHE_ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let cache_dir = temporary.path().join("isolated-carrier-cache"); + // Keep the carrier cache and all test objects outside the real user cache. + unsafe { env::set_var("GIT_REMOTE_E2EE_CACHE_DIR", &cache_dir) }; + + let carrier = temporary.path().join("carrier.git"); + let initialize_output = Command::new("git") + .args(["init", "--bare", "-q"]) + .arg(&carrier) + .output() + .unwrap(); + assert!(initialize_output.status.success()); + let carrier_url = format!("e2ee::git+{}", carrier.display()); + exercise_read_only_helper( + &temporary.path().join("carrier-case"), + GitStorage::open(carrier.to_str().unwrap()).unwrap(), + &carrier, + &carrier_url, + ); +} + +#[test] +fn read_only_device_cannot_publish_recovery() { + let temporary = tempfile::tempdir().unwrap(); + let storage_path = temporary.path().join("storage"); + let reader_repo = temporary.path().join("reader"); + initialize_git(&reader_repo); + let owner = KeyFile::generate(); + let reader = KeyFile::generate_for_repository(owner.repository_root.clone()).unwrap(); + let storage = FilesystemStorage::new(&storage_path); + let owner_repository = EncryptedRepository::new(storage.clone(), owner); + owner_repository.initialize().unwrap(); + let admin_pin = temporary.path().join("owner.admin-state.json"); + owner_repository.pin_admin_state(&admin_pin).unwrap(); + owner_repository + .add_device( + reader.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + owner_repository.fetch_into(&reader_repo, "origin").unwrap(); + let (floor_id, _) = owner_repository.current_manifest().unwrap(); + let invalid_bytes = b"malformed but content-addressed manifest"; + let invalid_id = git_remote_e2ee::crypto::object_id(invalid_bytes); + storage + .put_object_if_absent(ObjectKind::Manifest, &invalid_id, invalid_bytes) + .unwrap(); + storage + .compare_and_swap_head(Some(&floor_id), &invalid_id) + .unwrap(); + + let read_only = EncryptedRepository::new(storage.clone(), reader); + let report = read_only + .recover( + &reader_repo, + "origin", + RecoveryOptions { + publish: true, + base: None, + discard_newer: false, + accept_stale_floor: true, + }, + ) + .unwrap(); + assert_eq!(report.classification, RecoveryClass::Invalid); + assert!( + report + .blocked_reason + .as_deref() + .unwrap() + .contains("not an active writer") + ); + assert_eq!( + storage.read_head().unwrap().as_deref(), + Some(invalid_id.as_str()) + ); +} diff --git a/tests/remote_helper.rs b/tests/remote_helper.rs index 0886c6f..b40bbfe 100644 --- a/tests/remote_helper.rs +++ b/tests/remote_helper.rs @@ -1,9 +1,11 @@ use std::env; use std::fs; +use std::io::Write; use std::path::Path; -use std::process::{Command, Output}; +use std::process::{Command, Output, Stdio}; use git_remote_e2ee::crypto::KeyFile; +use git_remote_e2ee::policy::DeviceRoles; use git_remote_e2ee::repository::EncryptedRepository; use git_remote_e2ee::storage::FilesystemStorage; @@ -130,6 +132,97 @@ fn native_git_push_and_fetch_use_the_remote_helper() { ); } +#[test] +fn concurrent_helper_processes_preserve_the_client_state_floor() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let destination = temporary.path().join("destination"); + let remote_path = temporary.path().join("remote"); + let stale_remote = temporary.path().join("stale-remote"); + let key_path = temporary.path().join("repository.key.json"); + initialize_git(&source); + initialize_git(&destination); + + let key = KeyFile::generate(); + key.write_new(&key_path).unwrap(); + let encrypted = EncryptedRepository::new(FilesystemStorage::new(&remote_path), key); + encrypted.initialize().unwrap(); + configure_remote(&source, &remote_path, &key_path); + configure_remote(&destination, &remote_path, &key_path); + encrypted.fetch_into(&destination, "private").unwrap(); + + fs::write(source.join("note.md"), "first\n").unwrap(); + git(&source, &["add", "note.md"], false); + git(&source, &["commit", "-q", "-m", "first"], false); + git(&source, &["push", "private", "main"], true); + let stale_object = git(&source, &["rev-parse", "HEAD"], false); + copy_tree(&remote_path, &stale_remote); + + fs::write(source.join("note.md"), "second\n").unwrap(); + git(&source, &["add", "note.md"], false); + git(&source, &["commit", "-q", "-m", "second"], false); + let advertised_object = git(&source, &["rev-parse", "HEAD"], false); + git(&source, &["push", "private", "main"], true); + let (expected_head, expected_manifest) = encrypted.current_manifest().unwrap(); + + let current_url = format!("e2ee::{}", remote_path.display()); + let stale_url = format!("e2ee::{}", stale_remote.display()); + let helper = env!("CARGO_BIN_EXE_git-remote-e2ee"); + let mut children = Vec::new(); + for remote_url in [&stale_url, &stale_url, ¤t_url, ¤t_url] { + children.push( + Command::new(helper) + .arg("private") + .arg(remote_url) + .current_dir(&destination) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(), + ); + } + for (index, child) in children.iter_mut().enumerate() { + let object = if index < 2 { + &stale_object + } else { + &advertised_object + }; + let protocol = format!("capabilities\nlist\n\nfetch {object} refs/heads/main\n\n"); + child + .stdin + .take() + .unwrap() + .write_all(protocol.as_bytes()) + .unwrap(); + } + let mut current_successes = 0; + for (index, child) in children.into_iter().enumerate() { + let output = child.wait_with_output().unwrap(); + if index >= 2 { + assert!( + output.status.success(), + "current helper failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + current_successes += 1; + } else if !output.status.success() { + assert!( + String::from_utf8_lossy(&output.stderr).contains("rolled back"), + "stale helper failed unexpectedly: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + } + assert_eq!(current_successes, 2); + + let state_path = destination.join(".git/git-remote-e2ee/private/state.json"); + let state: serde_json::Value = serde_json::from_slice(&fs::read(state_path).unwrap()).unwrap(); + assert_eq!(state["generation"], expected_manifest.generation); + assert_eq!(state["imported_generation"], expected_manifest.generation); + assert_eq!(state["head_id"], expected_head); +} + #[test] fn native_git_dry_run_does_not_mutate_remote() { let temporary = tempfile::tempdir().unwrap(); @@ -293,17 +386,7 @@ fn native_git_fetch_rejects_same_generation_manifest_fork() { let fork = EncryptedRepository::new(FilesystemStorage::new(&fork_path), key); fork.initialize().unwrap(); let fork_head = fork.push_ref(&source, "refs/heads/main", false).unwrap(); - let fork_manifest = fork_path - .join("manifests") - .join(&fork_head[..2]) - .join(&fork_head); - let remote_manifest = remote_path - .join("manifests") - .join(&fork_head[..2]) - .join(&fork_head); - fs::create_dir_all(remote_manifest.parent().unwrap()).unwrap(); - fs::copy(fork_manifest, remote_manifest).unwrap(); - copy_tree(&fork_path.join("policies"), &remote_path.join("policies")); + copy_tree(&fork_path.join("manifests"), &remote_path.join("manifests")); fs::write(remote_path.join("HEAD"), format!("{fork_head}\n")).unwrap(); let output = git_output(&destination, &["fetch", "private"], true); @@ -314,3 +397,101 @@ fn native_git_fetch_rejects_same_generation_manifest_fork() { String::from_utf8_lossy(&output.stderr) ); } + +#[test] +fn native_git_fetch_pins_membership_changes_that_transfer_no_objects() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let destination = temporary.path().join("destination"); + let remote_path = temporary.path().join("remote"); + let key_path = temporary.path().join("repository.key.json"); + let admin_pin = temporary + .path() + .join("repository.key.json.admin-state.json"); + initialize_git(&source); + initialize_git(&destination); + + let key = KeyFile::generate(); + key.write_new(&key_path).unwrap(); + let extra = KeyFile::generate_for_repository(key.repository_root.clone()).unwrap(); + let encrypted = EncryptedRepository::new(FilesystemStorage::new(&remote_path), key); + encrypted.initialize().unwrap(); + encrypted.pin_admin_state(&admin_pin).unwrap(); + configure_remote(&source, &remote_path, &key_path); + configure_remote(&destination, &remote_path, &key_path); + + fs::write(source.join("note.md"), "first\n").unwrap(); + git(&source, &["add", "note.md"], false); + git(&source, &["commit", "-q", "-m", "first"], false); + git(&source, &["push", "private", "main"], true); + git(&destination, &["fetch", "private"], true); + let before_membership_change = encrypted.current_manifest().unwrap().0; + + // A membership change publishes a new generation without changing refs, so + // Git finds nothing to transfer and never sends the helper a fetch command. + encrypted + .add_device( + extra.public_device().unwrap(), + DeviceRoles::reader(), + &admin_pin, + ) + .unwrap(); + git(&destination, &["fetch", "private"], true); + + fs::write( + remote_path.join("HEAD"), + format!("{before_membership_change}\n"), + ) + .unwrap(); + let output = git_output(&destination, &["fetch", "private"], true); + assert!( + !output.status.success(), + "replaying the pre-change manifest must be rejected" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("rolled back"), + "stderr={}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn native_git_fetch_pins_branch_deletions_that_transfer_no_objects() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let destination = temporary.path().join("destination"); + let remote_path = temporary.path().join("remote"); + let key_path = temporary.path().join("repository.key.json"); + initialize_git(&source); + initialize_git(&destination); + + let key = KeyFile::generate(); + key.write_new(&key_path).unwrap(); + let encrypted = EncryptedRepository::new(FilesystemStorage::new(&remote_path), key); + encrypted.initialize().unwrap(); + configure_remote(&source, &remote_path, &key_path); + configure_remote(&destination, &remote_path, &key_path); + + fs::write(source.join("note.md"), "first\n").unwrap(); + git(&source, &["add", "note.md"], false); + git(&source, &["commit", "-q", "-m", "first"], false); + git(&source, &["branch", "feature"], false); + git(&source, &["push", "private", "main", "feature"], true); + git(&destination, &["fetch", "private"], true); + let before_deletion = encrypted.current_manifest().unwrap().0; + + git(&source, &["push", "private", "--delete", "feature"], true); + git(&destination, &["fetch", "--prune", "private"], true); + + fs::write(remote_path.join("HEAD"), format!("{before_deletion}\n")).unwrap(); + let output = git_output(&destination, &["fetch", "private"], true); + assert!( + !output.status.success(), + "replaying the pre-deletion manifest must be rejected" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("rolled back"), + "stderr={}", + String::from_utf8_lossy(&output.stderr) + ); +}