From 258de341f4e160639f6fcc2967fc360f331507e5 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 06:40:11 +0000 Subject: [PATCH 01/15] feat(TC-887): at-rest keyring, key store and migration Adds the S1 key hierarchy to tinycloud_core::at_rest. Nothing in the node constructs it yet. - hierarchy: W = HKDF(root), ARK sealed under W with AAD (ark_id, generation, W label), generation chaining (newest ARK opens older ones), KEK/NK derivation, per-space DEK wrap with length-prefixed AAD (space, dek_id, generation, ark_id). Key bytes only live in zeroizing buffers. - store: at_rest_key, at_rest_escrow and space_data_key rows (insert-or-ignore + re-read), and the KeyMirror trait for sealed-ARK and escrow copies next to the blocks (in-memory mirror for tests). - keyring: AtRestKeyring implements DataKeyProvider (current_key creates and converges, key never creates, object_name_key from ARK gen 1). Boot opens or provisions the ARK, restores rows from the mirror, never provisions over a store/mirror error or orphaned data keys, and boots locked when the root can't unseal. Escrow is kept to the configured holders (re-split on change, old bundles kept), the encrypt gate checks recipients, policy, pinned EFP and an ARK-MACed verification record, and the per-boot share intake runs the ceremony test-unlock (all three pairs) or locked-node recovery (two shares, reseal under the new root). Intake is commitment-checked, slot-capped and rate limited per boot. - migration m20261017_000000_at_rest_keys. --- Cargo.lock | 6 +- test/m1-realdata-e2e/Cargo.lock | 804 +++++++++++++++- test/w5-policy-runtime-node-e2e/Cargo.lock | 721 +++++++++++++- tinycloud-core/Cargo.toml | 6 + tinycloud-core/src/at_rest/envelope.rs | 2 +- tinycloud-core/src/at_rest/hierarchy.rs | 302 ++++++ tinycloud-core/src/at_rest/keyring.rs | 907 ++++++++++++++++++ tinycloud-core/src/at_rest/keyring/tests.rs | 597 ++++++++++++ tinycloud-core/src/at_rest/mod.rs | 6 + tinycloud-core/src/at_rest/store.rs | 335 +++++++ .../m20261017_000000_at_rest_keys.rs | 131 +++ tinycloud-core/src/migrations/mod.rs | 2 + tinycloud-core/src/models/at_rest_escrow.rs | 26 + tinycloud-core/src/models/at_rest_key.rs | 26 + tinycloud-core/src/models/mod.rs | 3 + tinycloud-core/src/models/space_data_key.rs | 24 + 16 files changed, 3824 insertions(+), 74 deletions(-) create mode 100644 tinycloud-core/src/at_rest/hierarchy.rs create mode 100644 tinycloud-core/src/at_rest/keyring.rs create mode 100644 tinycloud-core/src/at_rest/keyring/tests.rs create mode 100644 tinycloud-core/src/at_rest/store.rs create mode 100644 tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs create mode 100644 tinycloud-core/src/models/at_rest_escrow.rs create mode 100644 tinycloud-core/src/models/at_rest_key.rs create mode 100644 tinycloud-core/src/models/space_data_key.rs diff --git a/Cargo.lock b/Cargo.lock index 65fd4545..a21f7b87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10237,6 +10237,7 @@ version = "1.3.0" dependencies = [ "aes", "aes-gcm", + "age", "arrow", "async-std", "async-trait", @@ -10247,11 +10248,13 @@ dependencies = [ "futures", "hex", "hkdf", - "hmac 0.12.1", + "hmac", "iri-string", "k256", "libp2p", + "lru", "multihash-derive 0.9.1", + "parking_lot", "pin-project", "prometheus", "proptest", @@ -10269,6 +10272,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tokio", "tracing", diff --git a/test/m1-realdata-e2e/Cargo.lock b/test/m1-realdata-e2e/Cargo.lock index 34ed970a..e42fc069 100644 --- a/test/m1-realdata-e2e/Cargo.lock +++ b/test/m1-realdata-e2e/Cargo.lock @@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "087113bd50d9adce24850eed5d0476c7d199d532fce8fab5173650331e09033a" dependencies = [ "abnf-core", - "nom", + "nom 7.1.3", ] [[package]] @@ -18,7 +18,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c44e09c43ae1c368fb91a03a566472d0087c26cf7e1b9e8e289c14ede681dd7d" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -33,8 +33,8 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", - "generic-array", + "crypto-common 0.1.6", + "generic-array 0.14.9", ] [[package]] @@ -45,7 +45,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", "zeroize", ] @@ -64,6 +64,57 @@ dependencies = [ "zeroize", ] +[[package]] +name = "age" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd290633c2482479f70f6d1d96ae0e9f52c6a26cd5859edd47ee1fe33fc89f26" +dependencies = [ + "age-core", + "base64 0.22.1", + "bech32 0.11.1", + "chacha20poly1305", + "cipher", + "cookie-factory", + "hkdf", + "hmac", + "hpke", + "i18n-embed", + "i18n-embed-fl", + "lazy_static", + "ml-kem", + "nom 8.0.0", + "p256", + "pin-project", + "rand 0.8.5", + "rust-embed", + "scrypt", + "sha2 0.10.9", + "sha3", + "subtle", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "age-core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d4375964d1501e5f1b32aef2ead573913893ff238448d4e9fdf1522d828656" +dependencies = [ + "base64 0.22.1", + "bech32 0.11.1", + "chacha20poly1305", + "cookie-factory", + "hkdf", + "hpke", + "io_tee", + "nom 8.0.0", + "rand 0.8.5", + "secrecy", + "sha2 0.10.9", +] + [[package]] name = "ahash" version = "0.7.8" @@ -173,6 +224,15 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -200,7 +260,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -801,12 +861,27 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + [[package]] name = "bech32" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9ff0bbfd639f15c74af777d81383cf53efb7c93613f6cab67c6c11e05bbf8b" +[[package]] +name = "bech32" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -857,10 +932,22 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848" dependencies = [ - "funty", - "radium", + "funty 1.1.0", + "radium 0.6.2", + "tap", + "wyz 0.2.0", +] + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty 2.0.0", + "radium 0.7.0", "tap", - "wyz", + "wyz 0.5.1", ] [[package]] @@ -916,7 +1003,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.4.2", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -925,7 +1012,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "generic-array 0.14.9", ] [[package]] @@ -934,7 +1021,16 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.9", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array 0.4.10", ] [[package]] @@ -1075,7 +1171,7 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -1166,7 +1262,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "inout", "zeroize", ] @@ -1253,6 +1349,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-str" version = "0.4.3" @@ -1288,6 +1390,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1332,6 +1443,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -1392,7 +1512,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" dependencies = [ - "generic-array", + "generic-array 0.14.9", "subtle", ] @@ -1402,8 +1522,21 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.9", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96272c2ff28b807e09250b180ad1fb7889a3258f7455759b5c3c58b719467130" +dependencies = [ + "num-traits", "rand_core 0.6.4", + "serdect", "subtle", "zeroize", ] @@ -1414,11 +1547,20 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.9", "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array 0.4.10", +] + [[package]] name = "csv" version = "1.4.0" @@ -1478,7 +1620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -1639,7 +1781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] @@ -1678,7 +1820,7 @@ checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -1840,7 +1982,7 @@ version = "0.3.2" source = "git+https://github.com/tinycloudlabs/ssi#9d519692ab1b07b53d71f27ebad8473d64c338bf" dependencies = [ "async-trait", - "bech32", + "bech32 0.8.1", "bs58 0.4.0", "chrono", "iref", @@ -1894,7 +2036,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.9", ] [[package]] @@ -1905,10 +2047,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -2046,7 +2199,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff", - "generic-array", + "generic-array 0.14.9", "group", "hkdf", "pem-rfc7468 0.7.0", @@ -2054,6 +2207,21 @@ dependencies = [ "rand_core 0.6.4", "sec1", "subtle", + "tap", + "zeroize", +] + +[[package]] +name = "elliptic-curve-tools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de2b6fae800f08032a6ea32995b52925b1d451bff9d445c8ab2932323277faf" +dependencies = [ + "elliptic-curve", + "heapless", + "hex", + "multiexp", + "serde", "zeroize", ] @@ -2170,6 +2338,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ + "bitvec 1.1.1", "rand_core 0.6.4", "subtle", ] @@ -2194,6 +2363,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "find-crate" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a98bbaacea1c0eb6a0876280051b892eb73594fd90cf3b20e9c817029c57d2" +dependencies = [ + "toml 0.5.11", +] + [[package]] name = "find-msvc-tools" version = "0.1.8" @@ -2219,6 +2397,51 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fluent" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8137a6d5a2c50d6b0ebfcb9aaa91a28154e0a70605f112d30cb0cd4a78670477" +dependencies = [ + "fluent-bundle", + "unic-langid", +] + +[[package]] +name = "fluent-bundle" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01203cb8918f5711e73891b347816d932046f95f54207710bda99beaeb423bf4" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash", + "self_cell", + "smallvec", + "unic-langid", +] + +[[package]] +name = "fluent-langneg" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eebbe59450baee8282d71676f3bfed5689aeab00b27545e83e5f14b1195e8b0" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "fluent-syntax" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54f0d287c53ffd184d04d8677f590f4ac5379785529e5e08b1c8083acdd5c198" +dependencies = [ + "memchr", + "thiserror 2.0.18", +] + [[package]] name = "flume" version = "0.11.1" @@ -2242,6 +2465,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -2272,6 +2501,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.31" @@ -2403,6 +2638,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic-array" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb130435a959a8d525e6bca66ff6c40981a300ee96d70e3ef56f046556d614a3" +dependencies = [ + "rustversion", + "typenum", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -2521,6 +2766,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2557,7 +2811,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2565,6 +2819,9 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -2584,6 +2841,16 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -2641,6 +2908,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hpke" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4917627a14198c3603282c5158b815ad5534795451d3c074b53cf3cee0960b11" +dependencies = [ + "aead", + "aes-gcm", + "chacha20poly1305", + "digest 0.10.7", + "generic-array 0.14.9", + "hkdf", + "hmac", + "p256", + "rand_core 0.6.4", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "http" version = "0.2.12" @@ -2708,6 +2995,24 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d35805454dc9f8662a98d6d61886ffe26bd465f5960e0e55345c70d5c0d2a9" +dependencies = [ + "typenum", +] + +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -2870,6 +3175,72 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "i18n-config" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e06b90c8a0d252e203c94344b21e35a30f3a3a85dc7db5af8f8df9f3e0c63ef" +dependencies = [ + "basic-toml", + "log", + "serde", + "serde_derive", + "thiserror 1.0.69", + "unic-langid", +] + +[[package]] +name = "i18n-embed" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a217bbb075dcaefb292efa78897fc0678245ca67f265d12c351e42268fcb0305" +dependencies = [ + "arc-swap", + "fluent", + "fluent-langneg", + "fluent-syntax", + "i18n-embed-impl", + "intl-memoizer", + "log", + "parking_lot", + "rust-embed", + "thiserror 1.0.69", + "unic-langid", + "walkdir", +] + +[[package]] +name = "i18n-embed-fl" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602e6bd30c3db2749e13e38b363a3d98d9d41de1d8de7a79c31bb69e45b47cda" +dependencies = [ + "find-crate", + "fluent", + "fluent-syntax", + "i18n-config", + "i18n-embed", + "proc-macro-error3", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.114", + "unic-langid", +] + +[[package]] +name = "i18n-embed-impl" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2cc0e0523d1fe6fc2c6f66e5038624ea8091b3e7748b5e8e0c84b1698db6c2" +dependencies = [ + "find-crate", + "i18n-config", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -3081,7 +3452,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "generic-array", + "generic-array 0.14.9", ] [[package]] @@ -3093,6 +3464,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "intl-memoizer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310da2e345f5eb861e7a07ee182262e94975051db9e4223e909ba90f392f163f" +dependencies = [ + "type-map", + "unic-langid", +] + +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "io_tee" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3f7cef34251886990511df1c61443aa928499d598a9473929ab5a90a527304" + [[package]] name = "ipld-core" version = "0.4.2" @@ -3395,7 +3791,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -3408,6 +3804,16 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "kem" +version = "0.3.0-pre.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8645470337db67b01a7f966decf7d0bafedbae74147d33e641c67a91df239f" +dependencies = [ + "rand_core 0.6.4", + "zeroize", +] + [[package]] name = "langtag" version = "0.4.0" @@ -3749,6 +4155,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "m1-realdata-e2e" version = "0.0.0" @@ -3826,6 +4241,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3853,6 +4278,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ml-kem" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de49b3df74c35498c0232031bb7e85f9389f913e2796169c8ab47a53993a18f" +dependencies = [ + "hybrid-array 0.2.3", + "kem", + "rand_core 0.6.4", + "sha3", +] + [[package]] name = "mown" version = "0.2.2" @@ -3920,6 +4357,20 @@ dependencies = [ "data-encoding-macro", ] +[[package]] +name = "multiexp" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec2ce93a6f06ac6cae04c1da3f2a6a24fcfc1f0eb0b4e0f3d302f0df45326cb" +dependencies = [ + "ff", + "group", + "rand_core 0.6.4", + "rustversion", + "std-shims", + "zeroize", +] + [[package]] name = "multihash" version = "0.16.3" @@ -4042,6 +4493,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4051,6 +4511,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -4059,6 +4533,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "rand 0.8.5", ] [[package]] @@ -4077,6 +4552,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -4713,7 +5198,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -4725,7 +5210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -4835,6 +5320,16 @@ dependencies = [ "quote", ] +[[package]] +name = "proc-macro-error-attr3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e564d14133360e1ae169ffde5da25881b5fa47261665b8e5713c212c27799da" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "proc-macro-error2" version = "2.0.1" @@ -4847,6 +5342,18 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "proc-macro-error3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0d4471b3436c22106b21913b1dda531558918ae9b7ec55d58aa84b43552233" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "proc-macro2" version = "1.0.105" @@ -4968,6 +5475,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -5501,6 +6014,41 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust-embed" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19afa5b4b6a611de00bd1bdae6ae6f39084c9399f0679c3f52d8469cf335cc23" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d8afda6374eac59e066abee06d265247ebbaf3006cf878e2879e8356e34053" +dependencies = [ + "mime_guess", + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.114", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d84e8ba78bd384263e5922f084cbe1b081c3b7e69add59c8fb097b879ba968a" +dependencies = [ + "sha2 0.11.1", + "walkdir", +] + [[package]] name = "rust_decimal" version = "1.40.0" @@ -5512,6 +6060,12 @@ dependencies = [ "serde", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -5527,7 +6081,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -5700,6 +6254,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -5937,12 +6500,21 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der 0.7.10", - "generic-array", + "generic-array 0.14.9", "pkcs8 0.10.2", "subtle", "zeroize", ] +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -6222,6 +6794,16 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "serdect" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f42f67da2385b51a5f9652db9c93d78aeaf7610bf5ec366080b6de810604af53" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -6229,7 +6811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -6241,7 +6823,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -6253,10 +6835,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + [[package]] name = "sha3" version = "0.10.8" @@ -6426,6 +7019,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.5.4" @@ -6574,7 +7173,7 @@ dependencies = [ "futures-core", "futures-io", "futures-util", - "generic-array", + "generic-array 0.14.9", "hex", "hkdf", "hmac", @@ -7237,7 +7836,7 @@ version = "0.6.1" source = "git+https://github.com/tinycloudlabs/ssi#9d519692ab1b07b53d71f27ebad8473d64c338bf" dependencies = [ "base64 0.22.1", - "bitvec", + "bitvec 0.20.4", "chrono", "educe 0.4.23", "flate2", @@ -7427,6 +8026,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "std-shims" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "227c4f8561598188d0df96dbe749824576174bba278b5b6bb2eacff1066067d0" +dependencies = [ + "hashbrown 0.16.1", + "rustversion", + "spin 0.10.1", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -7497,6 +8107,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "0.1.2" @@ -7705,6 +8325,27 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinycloud-at-rest-escrow" +version = "0.1.0" +dependencies = [ + "age", + "age-core", + "base64 0.22.1", + "hex", + "hkdf", + "hmac", + "p256", + "rand 0.8.5", + "serde", + "serde_json", + "sha2 0.10.9", + "subtle", + "thiserror 2.0.18", + "vsss-rs", + "zeroize", +] + [[package]] name = "tinycloud-auth" version = "1.3.0" @@ -7744,7 +8385,9 @@ dependencies = [ "hmac", "k256", "libp2p", + "lru", "multihash-derive 0.9.1", + "parking_lot", "pin-project", "prometheus", "rand 0.8.5", @@ -7761,6 +8404,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tokio", "tracing", @@ -7846,6 +8490,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -8273,6 +8918,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash", +] + [[package]] name = "typenum" version = "1.19.0" @@ -8321,6 +8975,31 @@ dependencies = [ "version_check", ] +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "serde", + "tinystr", +] + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -8360,7 +9039,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "subtle", ] @@ -8466,6 +9145,36 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vsss-rs" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ec751bdcc8bda099e269b24cc6b4ad14f9ce8b0490c1599174070e792ecd70c" +dependencies = [ + "crypto-bigint 0.6.1", + "elliptic-curve", + "elliptic-curve-tools", + "generic-array 1.4.2", + "hex", + "hybrid-array 0.4.10", + "num", + "rand_core 0.6.4", + "serde", + "sha3", + "subtle", + "zeroize", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -8641,6 +9350,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -8994,6 +9712,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x25519-dalek" version = "2.0.1" @@ -9017,7 +9744,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "rusticata-macros", "thiserror 1.0.69", @@ -9170,6 +9897,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/test/w5-policy-runtime-node-e2e/Cargo.lock b/test/w5-policy-runtime-node-e2e/Cargo.lock index 2eac9317..f0309708 100644 --- a/test/w5-policy-runtime-node-e2e/Cargo.lock +++ b/test/w5-policy-runtime-node-e2e/Cargo.lock @@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "087113bd50d9adce24850eed5d0476c7d199d532fce8fab5173650331e09033a" dependencies = [ "abnf-core", - "nom", + "nom 7.1.3", ] [[package]] @@ -18,7 +18,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c44e09c43ae1c368fb91a03a566472d0087c26cf7e1b9e8e289c14ede681dd7d" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -33,8 +33,8 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", - "generic-array", + "crypto-common 0.1.6", + "generic-array 0.14.9", ] [[package]] @@ -46,6 +46,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -60,6 +61,58 @@ dependencies = [ "ctr", "ghash", "subtle", + "zeroize", +] + +[[package]] +name = "age" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd290633c2482479f70f6d1d96ae0e9f52c6a26cd5859edd47ee1fe33fc89f26" +dependencies = [ + "age-core", + "base64 0.22.1", + "bech32 0.11.1", + "chacha20poly1305", + "cipher", + "cookie-factory", + "hkdf", + "hmac", + "hpke", + "i18n-embed", + "i18n-embed-fl", + "lazy_static", + "ml-kem", + "nom 8.0.0", + "p256", + "pin-project", + "rand 0.8.5", + "rust-embed", + "scrypt", + "sha2 0.10.9", + "sha3", + "subtle", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "age-core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d4375964d1501e5f1b32aef2ead573913893ff238448d4e9fdf1522d828656" +dependencies = [ + "base64 0.22.1", + "bech32 0.11.1", + "chacha20poly1305", + "cookie-factory", + "hkdf", + "hpke", + "io_tee", + "nom 8.0.0", + "rand 0.8.5", + "secrecy", + "sha2 0.10.9", ] [[package]] @@ -153,7 +206,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -164,7 +217,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -182,6 +235,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -421,7 +483,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -970,12 +1032,27 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + [[package]] name = "bech32" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9ff0bbfd639f15c74af777d81383cf53efb7c93613f6cab67c6c11e05bbf8b" +[[package]] +name = "bech32" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -1106,7 +1183,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "generic-array 0.14.9", ] [[package]] @@ -1115,7 +1192,16 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.9", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array 0.4.10", ] [[package]] @@ -1418,7 +1504,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "inout", "zeroize", ] @@ -1516,6 +1602,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1571,6 +1663,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1684,7 +1785,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" dependencies = [ - "generic-array", + "generic-array 0.14.9", "subtle", ] @@ -1694,23 +1795,45 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.9", "rand_core 0.6.4", "subtle", "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96272c2ff28b807e09250b180ad1fb7889a3258f7455759b5c3c58b719467130" +dependencies = [ + "num-traits", + "rand_core 0.6.4", + "serdect", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.9", "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array 0.4.10", +] + [[package]] name = "csv" version = "1.4.0" @@ -1918,7 +2041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] @@ -1957,7 +2080,7 @@ checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -2130,7 +2253,7 @@ version = "0.3.2" source = "git+https://github.com/tinycloudlabs/ssi#9d519692ab1b07b53d71f27ebad8473d64c338bf" dependencies = [ "async-trait", - "bech32", + "bech32 0.8.1", "bs58 0.4.0", "chrono", "iref", @@ -2184,7 +2307,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.9", ] [[package]] @@ -2195,10 +2318,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -2331,7 +2465,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff", - "generic-array", + "generic-array 0.14.9", "group", "hkdf", "pem-rfc7468 0.7.0", @@ -2339,6 +2473,21 @@ dependencies = [ "rand_core 0.6.4", "sec1", "subtle", + "tap", + "zeroize", +] + +[[package]] +name = "elliptic-curve-tools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de2b6fae800f08032a6ea32995b52925b1d451bff9d445c8ab2932323277faf" +dependencies = [ + "elliptic-curve", + "heapless", + "hex", + "multiexp", + "serde", "zeroize", ] @@ -2397,7 +2546,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2455,6 +2604,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ + "bitvec 1.1.1", "rand_core 0.6.4", "subtle", ] @@ -2489,6 +2639,15 @@ dependencies = [ "libc", ] +[[package]] +name = "find-crate" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a98bbaacea1c0eb6a0876280051b892eb73594fd90cf3b20e9c817029c57d2" +dependencies = [ + "toml 0.5.11", +] + [[package]] name = "find-msvc-tools" version = "0.1.8" @@ -2525,6 +2684,51 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fluent" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8137a6d5a2c50d6b0ebfcb9aaa91a28154e0a70605f112d30cb0cd4a78670477" +dependencies = [ + "fluent-bundle", + "unic-langid", +] + +[[package]] +name = "fluent-bundle" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01203cb8918f5711e73891b347816d932046f95f54207710bda99beaeb423bf4" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash", + "self_cell", + "smallvec", + "unic-langid", +] + +[[package]] +name = "fluent-langneg" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eebbe59450baee8282d71676f3bfed5689aeab00b27545e83e5f14b1195e8b0" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "fluent-syntax" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54f0d287c53ffd184d04d8677f590f4ac5379785529e5e08b1c8083acdd5c198" +dependencies = [ + "memchr", + "thiserror 2.0.18", +] + [[package]] name = "flume" version = "0.11.1" @@ -2548,6 +2752,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -2715,6 +2925,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic-array" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb130435a959a8d525e6bca66ff6c40981a300ee96d70e3ef56f046556d614a3" +dependencies = [ + "rustversion", + "typenum", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -2837,6 +3057,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2873,7 +3102,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2881,6 +3110,9 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -2900,6 +3132,16 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -2957,6 +3199,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hpke" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4917627a14198c3603282c5158b815ad5534795451d3c074b53cf3cee0960b11" +dependencies = [ + "aead", + "aes-gcm", + "chacha20poly1305", + "digest 0.10.7", + "generic-array 0.14.9", + "hkdf", + "hmac", + "p256", + "rand_core 0.6.4", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "http" version = "0.2.12" @@ -3024,6 +3286,24 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d35805454dc9f8662a98d6d61886ffe26bd465f5960e0e55345c70d5c0d2a9" +dependencies = [ + "typenum", +] + +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -3187,6 +3467,72 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "i18n-config" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e06b90c8a0d252e203c94344b21e35a30f3a3a85dc7db5af8f8df9f3e0c63ef" +dependencies = [ + "basic-toml", + "log", + "serde", + "serde_derive", + "thiserror 1.0.69", + "unic-langid", +] + +[[package]] +name = "i18n-embed" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a217bbb075dcaefb292efa78897fc0678245ca67f265d12c351e42268fcb0305" +dependencies = [ + "arc-swap", + "fluent", + "fluent-langneg", + "fluent-syntax", + "i18n-embed-impl", + "intl-memoizer", + "log", + "parking_lot", + "rust-embed", + "thiserror 1.0.69", + "unic-langid", + "walkdir", +] + +[[package]] +name = "i18n-embed-fl" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602e6bd30c3db2749e13e38b363a3d98d9d41de1d8de7a79c31bb69e45b47cda" +dependencies = [ + "find-crate", + "fluent", + "fluent-syntax", + "i18n-config", + "i18n-embed", + "proc-macro-error3", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.114", + "unic-langid", +] + +[[package]] +name = "i18n-embed-impl" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2cc0e0523d1fe6fc2c6f66e5038624ea8091b3e7748b5e8e0c84b1698db6c2" +dependencies = [ + "find-crate", + "i18n-config", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -3398,7 +3744,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "generic-array", + "generic-array 0.14.9", ] [[package]] @@ -3410,6 +3756,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "intl-memoizer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310da2e345f5eb861e7a07ee182262e94975051db9e4223e909ba90f392f163f" +dependencies = [ + "type-map", + "unic-langid", +] + +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "io_tee" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3f7cef34251886990511df1c61443aa928499d598a9473929ab5a90a527304" + [[package]] name = "ipld-core" version = "0.4.2" @@ -3468,7 +3839,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3735,6 +4106,16 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "kem" +version = "0.3.0-pre.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8645470337db67b01a7f966decf7d0bafedbae74147d33e641c67a91df239f" +dependencies = [ + "rand_core 0.6.4", + "zeroize", +] + [[package]] name = "langtag" version = "0.4.0" @@ -4093,6 +4474,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.3" @@ -4147,6 +4537,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -4174,6 +4574,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ml-kem" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de49b3df74c35498c0232031bb7e85f9389f913e2796169c8ab47a53993a18f" +dependencies = [ + "hybrid-array 0.2.3", + "kem", + "rand_core 0.6.4", + "sha3", +] + [[package]] name = "mown" version = "0.2.2" @@ -4241,6 +4653,20 @@ dependencies = [ "data-encoding-macro", ] +[[package]] +name = "multiexp" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec2ce93a6f06ac6cae04c1da3f2a6a24fcfc1f0eb0b4e0f3d302f0df45326cb" +dependencies = [ + "ff", + "group", + "rand_core 0.6.4", + "rustversion", + "std-shims", + "zeroize", +] + [[package]] name = "multihash" version = "0.16.3" @@ -4363,13 +4789,22 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4394,6 +4829,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "rand 0.8.5", ] [[package]] @@ -4419,6 +4855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", + "rand 0.8.5", ] [[package]] @@ -5120,6 +5557,16 @@ dependencies = [ "quote", ] +[[package]] +name = "proc-macro-error-attr3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e564d14133360e1ae169ffde5da25881b5fa47261665b8e5713c212c27799da" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "proc-macro-error2" version = "2.0.1" @@ -5132,6 +5579,18 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "proc-macro-error3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0d4471b3436c22106b21913b1dda531558918ae9b7ec55d58aa84b43552233" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "proc-macro2" version = "1.0.105" @@ -5265,7 +5724,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.36", - "socket2 0.5.10", + "socket2 0.6.1", "thiserror 2.0.18", "tokio", "tracing", @@ -5303,9 +5762,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5901,6 +6360,41 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust-embed" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19afa5b4b6a611de00bd1bdae6ae6f39084c9399f0679c3f52d8469cf335cc23" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d8afda6374eac59e066abee06d265247ebbaf3006cf878e2879e8356e34053" +dependencies = [ + "mime_guess", + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.114", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d84e8ba78bd384263e5922f084cbe1b081c3b7e69add59c8fb097b879ba968a" +dependencies = [ + "sha2 0.11.1", + "walkdir", +] + [[package]] name = "rust_decimal" version = "1.40.0" @@ -5938,7 +6432,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -5964,7 +6458,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6112,6 +6606,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -6355,12 +6858,21 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der 0.7.10", - "generic-array", + "generic-array 0.14.9", "pkcs8 0.10.2", "subtle", "zeroize", ] +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -6640,6 +7152,16 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "serdect" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f42f67da2385b51a5f9652db9c93d78aeaf7610bf5ec366080b6de810604af53" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -6675,6 +7197,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + [[package]] name = "sha3" version = "0.10.8" @@ -6844,6 +7377,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.5.4" @@ -6992,7 +7531,7 @@ dependencies = [ "futures-core", "futures-io", "futures-util", - "generic-array", + "generic-array 0.14.9", "hex", "hkdf", "hmac", @@ -7845,6 +8384,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "std-shims" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "227c4f8561598188d0df96dbe749824576174bba278b5b6bb2eacff1066067d0" +dependencies = [ + "hashbrown 0.16.1", + "rustversion", + "spin 0.10.1", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -8067,7 +8617,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.3", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8179,6 +8729,27 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinycloud-at-rest-escrow" +version = "0.1.0" +dependencies = [ + "age", + "age-core", + "base64 0.22.1", + "hex", + "hkdf", + "hmac", + "p256", + "rand 0.8.5", + "serde", + "serde_json", + "sha2 0.10.9", + "subtle", + "thiserror 2.0.18", + "vsss-rs", + "zeroize", +] + [[package]] name = "tinycloud-auth" version = "1.3.0" @@ -8206,6 +8777,7 @@ dependencies = [ name = "tinycloud-core" version = "1.3.0" dependencies = [ + "aes", "aes-gcm", "arrow", "async-trait", @@ -8215,9 +8787,13 @@ dependencies = [ "duckdb", "futures", "hex", + "hkdf", + "hmac", "k256", "libp2p", + "lru", "multihash-derive 0.9.1", + "parking_lot", "pin-project", "prometheus", "rand 0.8.5", @@ -8234,12 +8810,14 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tokio", "tracing", "ucan-capabilities-object", "unicode-normalization", "x25519-dalek", + "zeroize", ] [[package]] @@ -8318,6 +8896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -8745,6 +9324,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash", +] + [[package]] name = "typenum" version = "1.19.0" @@ -8793,6 +9381,31 @@ dependencies = [ "version_check", ] +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "serde", + "tinystr", +] + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -8838,7 +9451,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "subtle", ] @@ -8944,6 +9557,26 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vsss-rs" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ec751bdcc8bda099e269b24cc6b4ad14f9ce8b0490c1599174070e792ecd70c" +dependencies = [ + "crypto-bigint 0.6.1", + "elliptic-curve", + "elliptic-curve-tools", + "generic-array 1.4.2", + "hex", + "hybrid-array 0.4.10", + "num", + "rand_core 0.6.4", + "serde", + "sha3", + "subtle", + "zeroize", +] + [[package]] name = "w5-policy-runtime-node-e2e" version = "0.0.0" @@ -8964,6 +9597,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -9133,6 +9776,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -9518,7 +10170,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "rusticata-macros", "thiserror 1.0.69", @@ -9681,6 +10333,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/tinycloud-core/Cargo.toml b/tinycloud-core/Cargo.toml index 1efb54dc..c9a62955 100644 --- a/tinycloud-core/Cargo.toml +++ b/tinycloud-core/Cargo.toml @@ -72,6 +72,10 @@ k256 = "0.13" hkdf = "0.12" hmac = "0.12" zeroize = "1" +# TC-887 S1: at-rest key hierarchy and escrow. +lru = "0.12" +parking_lot = "0.12" +tinycloud-at-rest-escrow = { path = "../tinycloud-at-rest-escrow" } prometheus = { version = "0.13.0", features = ["process"] } ryu-js = "0.2.2" # W1 (audit P1): full Unicode NFC normalization for policy capability path @@ -83,5 +87,7 @@ unicode-normalization = "0.1" sea-orm = { version = "1.1", features = ["runtime-tokio-rustls", "sqlx-sqlite"] } async-std = { version = "1", features = ["attributes"] } tokio.workspace = true +# Holder-side decryption of escrow shares in the at-rest keyring tests. +age = { version = "=0.12.1", default-features = false, features = ["armor"] } iri-string.workspace = true proptest = "1" diff --git a/tinycloud-core/src/at_rest/envelope.rs b/tinycloud-core/src/at_rest/envelope.rs index c1f401c6..4ea7d581 100644 --- a/tinycloud-core/src/at_rest/envelope.rs +++ b/tinycloud-core/src/at_rest/envelope.rs @@ -309,7 +309,7 @@ impl SegmentCipher { let mut aad = ctx; aad.extend_from_slice(header.as_bytes()); Self { - cipher: Aes256Gcm::new(k_obj.as_ref().into()), + cipher: Aes256Gcm::new((&*k_obj).into()), aad, segments: header.segment_count(), } diff --git a/tinycloud-core/src/at_rest/hierarchy.rs b/tinycloud-core/src/at_rest/hierarchy.rs new file mode 100644 index 00000000..83d74506 --- /dev/null +++ b/tinycloud-core/src/at_rest/hierarchy.rs @@ -0,0 +1,302 @@ +//! The at-rest key hierarchy below the node root R (TC-887 plan ยง1.1): +//! +//! ```text +//! W = HKDF-SHA256(R, info "tinycloud/at-rest/wrap/v1") never stored +//! ARK_g = 32 random bytes, sealed: nonce โ€– AES-256-GCM(W, ARK_g, AAD seal) +//! chain = nonce โ€– AES-256-GCM(HKDF(ARK_{g+1}, "โ€ฆ/ark-chain/v1"), ARK_g, AAD chain) +//! KEK_g = HKDF-SHA256(ARK_g, info "tinycloud/at-rest/kek/v1") +//! NK = HKDF-SHA256(ARK_1, info "tinycloud/at-rest/object-names/v1") +//! DEK = 32 random bytes per (space, dek_id): nonce โ€– AES-256-GCM(KEK_g, DEK, AAD dek) +//! ``` +//! +//! Every AAD starts with a length-prefixed domain string; variable fields are +//! length-prefixed (`lp(x) = u32 BE len โ€– x`). Nonces are random: each key +//! seals a handful of values, far below AES-GCM's random-nonce bound. + +use aes_gcm::{ + aead::{Aead, KeyInit, Payload}, + Aes256Gcm, Nonce, +}; +use hkdf::Hkdf; +use rand::RngCore; +use sha2::Sha256; +use tinycloud_at_rest_escrow::{Ark, ArkId}; +use zeroize::Zeroizing; + +use super::{DataKey, DekId, ObjectNameKey}; + +/// The label of the current W. A different label means a different W; rows +/// record which one sealed them. +pub const W_LABEL: &str = "tinycloud/at-rest/wrap/v1"; +const KEK_INFO: &[u8] = b"tinycloud/at-rest/kek/v1"; +const NK_INFO: &[u8] = b"tinycloud/at-rest/object-names/v1"; +const CHAIN_INFO: &[u8] = b"tinycloud/at-rest/ark-chain/v1"; +const SEAL_DOMAIN: &[u8] = b"tinycloud/at-rest/ark-seal/v1"; +const CHAIN_DOMAIN: &[u8] = b"tinycloud/at-rest/ark-chain-aad/v1"; +const DEK_DOMAIN: &[u8] = b"tinycloud/at-rest/dek/v1"; +const NONCE_LEN: usize = 12; + +/// An AEAD open failed: wrong key, wrong binding, or tampered bytes. +#[derive(Debug, thiserror::Error)] +#[error("at-rest key material does not authenticate")] +pub struct Unauthentic; + +fn lp(out: &mut Vec, bytes: &[u8]) { + let len = u32::try_from(bytes.len()).expect("field shorter than 4 GiB"); + out.extend_from_slice(&len.to_be_bytes()); + out.extend_from_slice(bytes); +} + +fn hkdf32(ikm: &[u8], info: &[u8]) -> Zeroizing<[u8; 32]> { + let mut out = Zeroizing::new([0u8; 32]); + Hkdf::::new(None, ikm) + .expand(info, out.as_mut()) + .expect("32 bytes is a valid HKDF-SHA256 length"); + out +} + +fn seal(key: &[u8; 32], aad: &[u8], plaintext: &[u8]) -> Vec { + let cipher = Aes256Gcm::new_from_slice(key).expect("32-byte key"); + let mut nonce = [0u8; NONCE_LEN]; + rand::rngs::OsRng.fill_bytes(&mut nonce); + let ct = cipher + .encrypt( + &Nonce::from(nonce), + Payload { + msg: plaintext, + aad, + }, + ) + .expect("AES-GCM encryption of 32 bytes cannot fail"); + let mut out = Vec::with_capacity(NONCE_LEN + ct.len()); + out.extend_from_slice(&nonce); + out.extend_from_slice(&ct); + out +} + +fn open32(key: &[u8; 32], aad: &[u8], sealed: &[u8]) -> Result, Unauthentic> { + if sealed.len() != NONCE_LEN + 32 + 16 { + return Err(Unauthentic); + } + let cipher = Aes256Gcm::new_from_slice(key).expect("32-byte key"); + let (nonce, ct) = sealed.split_at(NONCE_LEN); + let nonce: [u8; NONCE_LEN] = nonce.try_into().expect("length checked above"); + let plain = Zeroizing::new( + cipher + .decrypt(&Nonce::from(nonce), Payload { msg: ct, aad }) + .map_err(|_| Unauthentic)?, + ); + let mut out = Zeroizing::new([0u8; 32]); + out.copy_from_slice(&plain); + Ok(out) +} + +/// W, derived from the node root. Never stored. +pub struct WrapKey(Zeroizing<[u8; 32]>); + +impl WrapKey { + pub fn derive(root: &[u8]) -> Self { + Self(hkdf32(root, W_LABEL.as_bytes())) + } +} + +fn seal_aad(ark_id: &ArkId, generation: u32, w_label: &str) -> Vec { + let mut aad = Vec::with_capacity(96); + lp(&mut aad, SEAL_DOMAIN); + lp(&mut aad, &ark_id.0); + aad.extend_from_slice(&generation.to_be_bytes()); + lp(&mut aad, w_label.as_bytes()); + aad +} + +pub fn seal_ark(w: &WrapKey, ark: &Ark, ark_id: &ArkId, generation: u32) -> Vec { + seal(&w.0, &seal_aad(ark_id, generation, W_LABEL), ark.expose()) +} + +pub fn open_ark( + w: &WrapKey, + sealed: &[u8], + ark_id: &ArkId, + generation: u32, + w_label: &str, +) -> Result { + if w_label != W_LABEL { + return Err(Unauthentic); + } + open32(&w.0, &seal_aad(ark_id, generation, w_label), sealed).map(Ark::from_zeroizing) +} + +fn chain_aad(older_id: &ArkId, older_generation: u32, newer_id: &ArkId) -> Vec { + let mut aad = Vec::with_capacity(96); + lp(&mut aad, CHAIN_DOMAIN); + lp(&mut aad, &older_id.0); + aad.extend_from_slice(&older_generation.to_be_bytes()); + lp(&mut aad, &newer_id.0); + aad +} + +/// Seal generation g's ARK under generation g+1's chain key, so escrowing +/// the newest ARK recovers every older one. +pub fn chain_ark( + newer: &Ark, + newer_id: &ArkId, + older: &Ark, + older_id: &ArkId, + older_generation: u32, +) -> Vec { + let key = hkdf32(newer.expose(), CHAIN_INFO); + seal( + &key, + &chain_aad(older_id, older_generation, newer_id), + older.expose(), + ) +} + +pub fn open_chain( + newer: &Ark, + newer_id: &ArkId, + chain: &[u8], + older_id: &ArkId, + older_generation: u32, +) -> Result { + let key = hkdf32(newer.expose(), CHAIN_INFO); + open32( + &key, + &chain_aad(older_id, older_generation, newer_id), + chain, + ) + .map(Ark::from_zeroizing) +} + +/// KEK_g. Wraps the DEKs created while generation g was current. +pub struct Kek(Zeroizing<[u8; 32]>); + +impl Kek { + pub fn derive(ark: &Ark) -> Self { + Self(hkdf32(ark.expose(), KEK_INFO)) + } +} + +/// NK from ARK generation 1; constant for the deployment. +pub fn object_name_key(ark_1: &Ark) -> ObjectNameKey { + let nk = hkdf32(ark_1.expose(), NK_INFO); + ObjectNameKey::new(*nk) +} + +fn dek_aad(space: &str, dek_id: DekId, generation: u32, ark_id: &ArkId) -> Vec { + let mut aad = Vec::with_capacity(64 + space.len()); + lp(&mut aad, DEK_DOMAIN); + lp(&mut aad, space.as_bytes()); + aad.extend_from_slice(&dek_id.get().to_be_bytes()); + aad.extend_from_slice(&generation.to_be_bytes()); + lp(&mut aad, &ark_id.0); + aad +} + +/// A fresh random DEK, wrapped. Returns the wrapped bytes and the key. +pub fn new_dek( + kek: &Kek, + space: &str, + dek_id: DekId, + generation: u32, + ark_id: &ArkId, +) -> (Vec, DataKey) { + let mut bytes = Zeroizing::new([0u8; 32]); + rand::rngs::OsRng.fill_bytes(bytes.as_mut()); + let wrapped = seal( + &kek.0, + &dek_aad(space, dek_id, generation, ark_id), + bytes.as_ref(), + ); + (wrapped, DataKey::new(dek_id, *bytes)) +} + +pub fn unwrap_dek( + kek: &Kek, + wrapped: &[u8], + space: &str, + dek_id: DekId, + generation: u32, + ark_id: &ArkId, +) -> Result { + let bytes = open32(&kek.0, &dek_aad(space, dek_id, generation, ark_id), wrapped)?; + Ok(DataKey::new(dek_id, *bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ark_seal_opens_only_under_the_same_root_and_binding() { + let w = WrapKey::derive(&[7u8; 64]); + let ark = Ark::generate(); + let id = ArkId([1; 16]); + let sealed = seal_ark(&w, &ark, &id, 1); + assert!(open_ark(&w, &sealed, &id, 1, W_LABEL).unwrap().ct_eq(&ark)); + + let other_root = WrapKey::derive(&[8u8; 64]); + assert!(open_ark(&other_root, &sealed, &id, 1, W_LABEL).is_err()); + assert!(open_ark(&w, &sealed, &ArkId([2; 16]), 1, W_LABEL).is_err()); + assert!(open_ark(&w, &sealed, &id, 2, W_LABEL).is_err()); + assert!(open_ark(&w, &sealed, &id, 1, "tinycloud/at-rest/wrap/v2").is_err()); + let mut flipped = sealed.clone(); + flipped[20] ^= 1; + assert!(open_ark(&w, &flipped, &id, 1, W_LABEL).is_err()); + assert!(open_ark(&w, &sealed[..sealed.len() - 1], &id, 1, W_LABEL).is_err()); + } + + #[test] + fn chain_opens_older_generation_from_newer_only() { + let (old, old_id) = (Ark::generate(), ArkId([1; 16])); + let (new, new_id) = (Ark::generate(), ArkId([2; 16])); + let chain = chain_ark(&new, &new_id, &old, &old_id, 1); + assert!(open_chain(&new, &new_id, &chain, &old_id, 1) + .unwrap() + .ct_eq(&old)); + assert!(open_chain(&old, &new_id, &chain, &old_id, 1).is_err()); + assert!(open_chain(&new, &new_id, &chain, &old_id, 2).is_err()); + assert!(open_chain(&new, &old_id, &chain, &old_id, 1).is_err()); + } + + #[test] + fn dek_wrap_binds_space_id_generation_and_ark() { + let ark = Ark::generate(); + let id = ArkId([3; 16]); + let kek = Kek::derive(&ark); + let (wrapped, dek) = new_dek(&kek, "space-a", DekId::new(1), 1, &id); + let back = unwrap_dek(&kek, &wrapped, "space-a", DekId::new(1), 1, &id).unwrap(); + assert_eq!(back.bytes(), dek.bytes()); + assert_eq!(back.id(), DekId::new(1)); + + // A row moved to another space, id, generation or ARK fails. + assert!(unwrap_dek(&kek, &wrapped, "space-b", DekId::new(1), 1, &id).is_err()); + assert!(unwrap_dek(&kek, &wrapped, "space-a", DekId::new(2), 1, &id).is_err()); + assert!(unwrap_dek(&kek, &wrapped, "space-a", DekId::new(1), 2, &id).is_err()); + assert!(unwrap_dek(&kek, &wrapped, "space-a", DekId::new(1), 1, &ArkId([4; 16])).is_err()); + assert!(unwrap_dek( + &Kek::derive(&Ark::generate()), + &wrapped, + "space-a", + DekId::new(1), + 1, + &id + ) + .is_err()); + // Length-prefixing: "ab"+"c" and "a"+"bc" style shifts can't collide. + assert_ne!( + dek_aad("ab", DekId::new(1), 1, &id), + dek_aad("a", DekId::new(1), 1, &id) + ); + } + + #[test] + fn derived_keys_are_independent() { + let ark = Ark::generate(); + let kek = Kek::derive(&ark); + let nk = object_name_key(&ark); + assert_ne!(&*kek.0, nk.bytes()); + assert_ne!(&*kek.0, ark.expose()); + assert_ne!(nk.bytes(), &*hkdf32(ark.expose(), CHAIN_INFO)); + } +} diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs new file mode 100644 index 00000000..c05848da --- /dev/null +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -0,0 +1,907 @@ +//! The node's at-rest keyring (TC-887 S1): loads or provisions the ARK under +//! W, keeps every generation, serves per-space DEKs ([`DataKeyProvider`]), +//! maintains the escrow bundle, decides the encrypt gate, and runs the +//! in-TEE share intake for recovery ("locked, not lost") and the ceremony +//! test-unlock. + +use std::collections::BTreeMap; +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use lru::LruCache; +use parking_lot::{Mutex, RwLock}; +use sea_orm::DbErr; +use tinycloud_at_rest_escrow::{ + Ark, ArkId, Digest32, Efp, EscrowBundle, EscrowError, HolderRecipient, RecipientPolicy, + RecoveryIdentity, SharePayload, VerificationRecord, +}; + +use super::hierarchy::{ + chain_ark, new_dek, object_name_key, open_ark, open_chain, seal_ark, unwrap_dek, Kek, WrapKey, + W_LABEL, +}; +use super::store::{ + ark_mirror_name, escrow_mirror_name, AtRestStore, KeyMirror, MirroredArk, ARK_MIRROR_PREFIX, + MIRROR_PREFIX, +}; +use super::{DataKey, DataKeyError, DataKeyProvider, DekId, ObjectNameKey}; +use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; + +const DEK_CACHE: usize = 4096; +/// Share submissions per boot: a burst of this many, then one per refill. +const SHARE_BURST: u32 = 10; +const SHARE_REFILL: Duration = Duration::from_secs(6); + +/// How the node runs its keyring. +#[derive(Clone, Debug, Default)] +pub struct KeyringConfig { + /// Make ARK generation 1 if none exists. + pub provision: bool, + /// Escrow holders pinned in config; empty means no escrow. + pub holders: Vec, + /// Hardware-only by default; software (X25519) holders need the explicit + /// test/self-host opt-in. + pub recipient_policy: RecipientPolicy, + /// `at_rest.escrow.verified_fp`: the bundle a ceremony verified. + pub pinned_efp: Option, + /// The TEE identity written into verification records (informational). + pub app_id: Option, + pub compose_hash: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum AtRestError { + #[error("at-rest key store: {0}")] + Store(#[from] DbErr), + #[error("at-rest key mirror: {0}")] + Mirror(String), + #[error("at-rest key state is inconsistent: {0}")] + Inconsistent(&'static str), + #[error("at-rest escrow: {0}")] + Escrow(#[from] EscrowError), + #[error("at-rest keys are locked")] + Locked, + #[error("at-rest keys are not provisioned")] + NotProvisioned, +} + +/// Why `encrypt` may not be enabled. Every variant is an operator action. +#[derive(Debug, PartialEq, Eq, thiserror::Error)] +pub enum GateRefusal { + #[error("at-rest keys are locked; recover with two escrow shares first")] + Locked, + #[error("at-rest keys are not provisioned")] + NotProvisioned, + #[error("no escrow recipients are configured; set at_rest.escrow.recipients (or, on a self-hosted Static-root node, acknowledge at_rest.escrow_none)")] + NoEscrow, + #[error("the active escrow bundle's recipients differ from the configured recipients")] + RecipientsMismatch, + #[error("the escrow has software (X25519) holders, which only an explicit test/self-host opt-in allows")] + SoftwareRecipients, + #[error("at_rest.escrow.verified_fp is unset; run the escrow ceremony and pin the verified fingerprint {efp}")] + NotPinned { efp: String }, + #[error("at_rest.escrow.verified_fp ({pinned}) is not the active escrow bundle ({efp}); re-run the ceremony")] + PinMismatch { pinned: String, efp: String }, + #[error("escrow {efp} has no valid verification record; run the ceremony test-unlock")] + NotVerified { efp: String }, +} + +/// How a passing gate was satisfied. +#[derive(Debug, PartialEq, Eq)] +pub enum GatePass { + Verified { + efp: Efp, + }, + /// Self-host with `escrow_none` acknowledged: no escrow exists. + Unescrowed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum KeyringState { + NotProvisioned, + Locked, + Unlocked, +} + +/// Public facts for `/info` and the recovery endpoint. +#[derive(Clone, Debug)] +pub struct EscrowView { + pub state: KeyringState, + pub ark_id: ArkId, + pub generation: u32, + pub efp: Efp, + pub canary_fp: Digest32, + pub verified: bool, + pub recovery_recipient: String, + pub shares_held: Vec, + pub bundle: EscrowBundle, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum ShareOutcome { + /// Valid; waiting for more shares. + Held { shares_held: Vec }, + /// Unlocked node: all three shares rebuilt the ARK; record written. + Verified { efp: Efp }, + /// Locked node: two shares recovered the ARK; resealed and unlocked. + Unlocked, +} + +#[derive(Debug, thiserror::Error)] +pub enum ShareRefusal { + #[error("too many share submissions; retry later")] + RateLimited, + #[error("no escrow bundle to recover against: {0}")] + NoBundle(&'static str), + #[error("share rejected: {0}")] + Invalid(#[from] EscrowError), + #[error("{0}")] + Keyring(#[from] AtRestError), +} + +struct Generation { + ark_id: ArkId, + ark: Ark, + kek: Kek, +} + +struct Keys { + gens: BTreeMap, + nk: ObjectNameKey, +} + +impl Keys { + fn newest(&self) -> (u32, &Generation) { + let (g, gen) = self.gens.last_key_value().expect("at least generation 1"); + (*g, gen) + } +} + +enum Phase { + NotProvisioned, + Locked { generation: u32, ark_id: ArkId }, + Unlocked(Arc), +} + +struct ShareIntake { + identity: RecoveryIdentity, + efp: Option, + held: [Option; 3], + tokens: u32, + refilled: Instant, +} + +impl ShareIntake { + fn new() -> Self { + Self { + identity: RecoveryIdentity::generate(), + efp: None, + held: [None, None, None], + tokens: SHARE_BURST, + refilled: Instant::now(), + } + } + + fn take_token(&mut self) -> bool { + let earned = (self.refilled.elapsed().as_secs() / SHARE_REFILL.as_secs()) as u32; + if earned > 0 { + self.tokens = (self.tokens + earned).min(SHARE_BURST); + self.refilled += SHARE_REFILL * earned; + } + if self.tokens == 0 { + return false; + } + self.tokens -= 1; + true + } + + fn held_indices(&self) -> Vec { + self.held.iter().flatten().map(|s| s.index).collect() + } + + fn clear(&mut self) { + self.held = [None, None, None]; + } +} + +fn now_rfc3339() -> String { + time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .expect("RFC 3339 formats any UTC time") +} + +fn mirror_err(e: impl std::fmt::Display) -> AtRestError { + AtRestError::Mirror(e.to_string()) +} + +fn ark_id_of(row: &at_rest_key::Model) -> Result { + row.ark_id + .as_slice() + .try_into() + .map(ArkId) + .map_err(|_| AtRestError::Inconsistent("ark_id is not 16 bytes")) +} + +fn backend(e: impl std::error::Error + Send + Sync + 'static) -> DataKeyError { + DataKeyError::Backend(Box::new(e)) +} + +/// Open every generation from the rows: the newest under W, older ones via +/// their chain (or W). `Ok(None)` means the newest doesn't open under W. +fn unlock_rows(w: &WrapKey, rows: &[at_rest_key::Model]) -> Result, AtRestError> { + for (i, row) in rows.iter().enumerate() { + if row.generation != i as i32 + 1 { + return Err(AtRestError::Inconsistent("ARK generations are not 1..n")); + } + } + let newest = rows.last().expect("caller checked non-empty"); + let newest_id = ark_id_of(newest)?; + let Ok(ark) = open_ark( + w, + &newest.sealed, + &newest_id, + newest.generation as u32, + &newest.w_label, + ) else { + return Ok(None); + }; + keys_from_newest(w, rows, ark).map(Some) +} + +fn keys_from_newest( + w: &WrapKey, + rows: &[at_rest_key::Model], + newest_ark: Ark, +) -> Result { + let mut gens = BTreeMap::new(); + let mut newer: Option<(Ark, ArkId)> = None; + let mut ark = Some(newest_ark); + for row in rows.iter().rev() { + let id = ark_id_of(row)?; + let g = row.generation as u32; + let this = match ark.take() { + Some(a) => a, + None => { + let (n_ark, n_id) = newer.as_ref().expect("set after the newest"); + row.chain + .as_deref() + .and_then(|c| open_chain(n_ark, n_id, c, &id, g).ok()) + .or_else(|| open_ark(w, &row.sealed, &id, g, &row.w_label).ok()) + .ok_or(AtRestError::Inconsistent( + "an older ARK generation opens neither by chain nor by W", + ))? + } + }; + newer = Some((this.clone(), id)); + gens.insert( + g, + Generation { + ark_id: id, + kek: Kek::derive(&this), + ark: this, + }, + ); + } + let nk = object_name_key(&gens.get(&1).expect("generation 1 exists").ark); + Ok(Keys { gens, nk }) +} + +pub struct AtRestKeyring { + store: AtRestStore, + mirror: Arc, + w: WrapKey, + config: KeyringConfig, + phase: RwLock, + deks: Mutex>, + current: Mutex>, + intake: tokio::sync::Mutex, + dek_create: tokio::sync::Mutex<()>, +} + +impl AtRestKeyring { + /// Load the keyring at boot. + /// + /// - Rows open under this root's W: unlocked. + /// - Rows exist but don't open: **locked** (data intact; recover with two + /// shares). + /// - No rows: restore them from the mirror if it has any; otherwise + /// provision generation 1 if `config.provision`, else not provisioned. + /// A store or mirror error fails the call; it never provisions over + /// uncertainty, and never when data keys exist without an ARK. + pub async fn open( + store: AtRestStore, + mirror: Arc, + root: &[u8], + config: KeyringConfig, + ) -> Result { + let w = WrapKey::derive(root); + let mut rows = store.keys().await?; + if rows.is_empty() { + rows = Self::restore_rows(&store, &*mirror).await?; + } + if rows.is_empty() { + if store.has_space_keys().await? { + return Err(AtRestError::Inconsistent( + "data keys exist but no ARK row or mirrored ARK was found", + )); + } + if config.provision { + let ark = Ark::generate(); + let ark_id = ArkId::generate(); + store + .insert_key(at_rest_key::Model { + generation: 1, + ark_id: ark_id.0.to_vec(), + w_label: W_LABEL.into(), + sealed: seal_ark(&w, &ark, &ark_id, 1), + chain: None, + created_at: now_rfc3339(), + }) + .await?; + // Another process may have won the insert; use what's stored. + rows = store.keys().await?; + tracing::info!(generation = 1, %ark_id, "at-rest root key provisioned"); + } + } + let phase = if rows.is_empty() { + Phase::NotProvisioned + } else { + match unlock_rows(&w, &rows)? { + Some(keys) => { + for row in &rows { + Self::mirror_row(&*mirror, row).await?; + } + Phase::Unlocked(Arc::new(keys)) + } + None => { + let newest = rows.last().expect("non-empty"); + tracing::warn!( + generation = newest.generation, + "at-rest keys locked: this node's root cannot unseal the ARK" + ); + Phase::Locked { + generation: newest.generation as u32, + ark_id: ark_id_of(newest)?, + } + } + } + }; + let keyring = Self { + store, + mirror, + w, + config, + phase: RwLock::new(phase), + deks: Mutex::new(LruCache::new( + NonZeroUsize::new(DEK_CACHE).expect("non-zero"), + )), + current: Mutex::new(LruCache::new( + NonZeroUsize::new(DEK_CACHE).expect("non-zero"), + )), + intake: tokio::sync::Mutex::new(ShareIntake::new()), + dek_create: tokio::sync::Mutex::new(()), + }; + if keyring.state() == KeyringState::Unlocked { + keyring.ensure_escrow().await?; + } + Ok(keyring) + } + + async fn restore_rows( + store: &AtRestStore, + mirror: &dyn KeyMirror, + ) -> Result, AtRestError> { + let mut names = mirror.list(ARK_MIRROR_PREFIX).await.map_err(mirror_err)?; + if names.is_empty() { + return Ok(Vec::new()); + } + names.sort(); + for name in names { + let bytes = + mirror + .get(&name) + .await + .map_err(mirror_err)? + .ok_or(AtRestError::Inconsistent( + "mirrored ARK vanished while listing", + ))?; + let row = serde_json::from_slice::(&bytes) + .map_err(mirror_err)? + .into_row() + .map_err(mirror_err)?; + store.insert_key(row).await?; + } + tracing::warn!("at-rest ARK rows were missing from the database; restored from the mirror"); + Ok(store.keys().await?) + } + + async fn mirror_row( + mirror: &dyn KeyMirror, + row: &at_rest_key::Model, + ) -> Result<(), AtRestError> { + let name = ark_mirror_name(row.generation as u32, &hex::encode(&row.ark_id)); + let body = serde_json::to_vec(&MirroredArk::from_row(row)).expect("serializable"); + if mirror.get(&name).await.map_err(mirror_err)?.as_deref() != Some(body.as_slice()) { + mirror.put(&name, body).await.map_err(mirror_err)?; + } + Ok(()) + } + + pub fn state(&self) -> KeyringState { + match &*self.phase.read() { + Phase::NotProvisioned => KeyringState::NotProvisioned, + Phase::Locked { .. } => KeyringState::Locked, + Phase::Unlocked(_) => KeyringState::Unlocked, + } + } + + fn keys(&self) -> Result, AtRestError> { + match &*self.phase.read() { + Phase::Unlocked(keys) => Ok(keys.clone()), + Phase::Locked { .. } => Err(AtRestError::Locked), + Phase::NotProvisioned => Err(AtRestError::NotProvisioned), + } + } + + fn parse_bundle(row: &at_rest_escrow::Model) -> Result { + let bundle: EscrowBundle = serde_json::from_str(&row.bundle) + .map_err(|_| AtRestError::Inconsistent("stored escrow bundle is not valid JSON"))?; + bundle.validate()?; + if bundle.fingerprint().to_string() != row.efp { + return Err(AtRestError::Inconsistent( + "stored escrow bundle does not match its fingerprint", + )); + } + Ok(bundle) + } + + /// Make sure the newest generation is escrowed to exactly the configured + /// holders. A different holder list re-splits the same ARK into a new + /// bundle (new EFP, so a new ceremony); old bundles are kept inactive. + async fn ensure_escrow(&self) -> Result, AtRestError> { + if self.config.holders.is_empty() { + return Ok(None); + } + let keys = self.keys()?; + let (g, gen) = keys.newest(); + if let Some(row) = self.store.active_escrow(g as i32).await? { + let bundle = Self::parse_bundle(&row)?; + if bundle.holders() == self.config.holders && bundle.ark_id == gen.ark_id { + self.mirror_escrow(&row).await?; + return Ok(Some(bundle)); + } + } + let bundle = EscrowBundle::create( + &gen.ark, + gen.ark_id, + g, + &self.config.holders, + self.config.recipient_policy, + )?; + let row = at_rest_escrow::Model { + efp: bundle.fingerprint().to_string(), + generation: g as i32, + bundle: serde_json::to_string(&bundle).expect("serializable"), + active: true, + record: None, + created_at: now_rfc3339(), + }; + self.store.activate_escrow(row.clone()).await?; + self.mirror_escrow(&row).await?; + tracing::warn!( + generation = g, + efp = %row.efp, + "at-rest escrow bundle created; the escrow ceremony must verify it before encryption can be enabled" + ); + Ok(Some(bundle)) + } + + async fn mirror_escrow(&self, row: &at_rest_escrow::Model) -> Result<(), AtRestError> { + let name = escrow_mirror_name(&row.efp); + if self.mirror.get(&name).await.map_err(mirror_err)?.is_none() { + self.mirror + .put(&name, row.bundle.clone().into_bytes()) + .await + .map_err(mirror_err)?; + } + Ok(()) + } + + /// The pinned bundle, from the database or else the mirror. Only a bundle + /// whose fingerprint equals the pin is returned. + async fn pinned_bundle(&self) -> Result, AtRestError> { + let Some(pinned) = self.config.pinned_efp else { + return Ok(None); + }; + let efp = pinned.to_string(); + if let Some(row) = self.store.escrow(&efp).await? { + return Self::parse_bundle(&row).map(Some); + } + let Some(bytes) = self + .mirror + .get(&escrow_mirror_name(&efp)) + .await + .map_err(mirror_err)? + else { + return Ok(None); + }; + let bundle: EscrowBundle = serde_json::from_slice(&bytes) + .map_err(|_| AtRestError::Inconsistent("mirrored escrow bundle is not valid JSON"))?; + bundle.validate()?; + Ok((bundle.fingerprint() == pinned).then_some(bundle)) + } + + /// The bundle shares are checked against: the active one when unlocked, + /// the pinned one when locked (storage is untrusted, the pin is attested). + async fn intake_bundle(&self) -> Result { + let locked = match &*self.phase.read() { + Phase::NotProvisioned => { + return Err(ShareRefusal::NoBundle("keys are not provisioned")) + } + Phase::Locked { generation, ark_id } => Some((*generation, *ark_id)), + Phase::Unlocked(_) => None, + }; + match locked { + Some((generation, ark_id)) => { + let bundle = self.pinned_bundle().await?.ok_or(ShareRefusal::NoBundle( + "no bundle matches at_rest.escrow.verified_fp", + ))?; + if bundle.generation != generation || bundle.ark_id != ark_id { + return Err(ShareRefusal::NoBundle( + "the pinned bundle escrows a different ARK generation", + )); + } + Ok(bundle) + } + None => self.ensure_escrow().await?.ok_or(ShareRefusal::NoBundle( + "no escrow recipients are configured", + )), + } + } + + /// Facts for `/info` and `GET /at-rest/recovery`. + pub async fn escrow_view(&self) -> Result, AtRestError> { + let state = self.state(); + let bundle = match self.intake_bundle().await { + Ok(b) => b, + Err(ShareRefusal::Keyring(e)) => return Err(e), + Err(_) => return Ok(None), + }; + let verified = match state { + KeyringState::Unlocked => self.verified_record(&bundle).await?.is_some(), + _ => false, + }; + let intake = self.intake.lock().await; + let shares_held = if intake.efp == Some(bundle.fingerprint()) { + intake.held_indices() + } else { + Vec::new() + }; + Ok(Some(EscrowView { + state, + ark_id: bundle.ark_id, + generation: bundle.generation, + efp: bundle.fingerprint(), + canary_fp: bundle.canary_fp, + verified, + recovery_recipient: intake.identity.recipient(), + shares_held, + bundle, + })) + } + + async fn verified_record( + &self, + bundle: &EscrowBundle, + ) -> Result, AtRestError> { + let keys = self.keys()?; + let Some(gen) = keys.gens.get(&bundle.generation) else { + return Ok(None); + }; + let Some(row) = self.store.escrow(&bundle.fingerprint().to_string()).await? else { + return Ok(None); + }; + let Some(json) = row.record else { + return Ok(None); + }; + let Ok(record) = serde_json::from_str::(&json) else { + return Ok(None); + }; + Ok(record.verify(&gen.ark, bundle).is_ok().then_some(record)) + } + + /// The encrypt gate. `escrow_none` is the explicit self-host + /// acknowledgement that no escrow exists; it only applies when no + /// recipients are configured. + pub async fn gate(&self, escrow_none: bool) -> Result { + match self.state() { + KeyringState::Locked => return Err(GateRefusal::Locked), + KeyringState::NotProvisioned => return Err(GateRefusal::NotProvisioned), + KeyringState::Unlocked => {} + } + if self.config.holders.is_empty() { + return if escrow_none { + Ok(GatePass::Unescrowed) + } else { + Err(GateRefusal::NoEscrow) + }; + } + let bundle = match self.ensure_escrow().await { + Ok(Some(b)) => b, + _ => return Err(GateRefusal::NoEscrow), + }; + if bundle.holders() != self.config.holders { + return Err(GateRefusal::RecipientsMismatch); + } + if bundle + .check_recipients(self.config.recipient_policy) + .is_err() + { + return Err(GateRefusal::SoftwareRecipients); + } + let efp = bundle.fingerprint(); + match self.config.pinned_efp { + None => { + return Err(GateRefusal::NotPinned { + efp: efp.to_string(), + }) + } + Some(p) if p != efp => { + return Err(GateRefusal::PinMismatch { + pinned: p.to_string(), + efp: efp.to_string(), + }) + } + Some(_) => {} + } + match self.verified_record(&bundle).await { + Ok(Some(_)) => Ok(GatePass::Verified { efp }), + _ => Err(GateRefusal::NotVerified { + efp: efp.to_string(), + }), + } + } + + /// Take one wrapped share (age file to this boot's recovery recipient). + /// + /// Each share is checked against its commitment before it occupies one of + /// three slots (one per index); submissions are rate limited per boot. + /// Share bytes are never logged. + pub async fn submit_share(&self, wrapped: &[u8]) -> Result { + let mut intake = self.intake.lock().await; + if !intake.take_token() { + return Err(ShareRefusal::RateLimited); + } + let share = intake.identity.open(wrapped)?; + let bundle = self.intake_bundle().await?; + bundle.verify_share(&share)?; + let efp = bundle.fingerprint(); + if intake.efp != Some(efp) { + intake.clear(); + intake.efp = Some(efp); + } + let index = share.index; + intake.held[usize::from(index) - 1] = Some(share); + tracing::info!(%efp, index, "at-rest escrow share accepted"); + + let held: Vec = intake.held.iter().flatten().cloned().collect(); + let outcome = match self.state() { + KeyringState::Locked if held.len() >= 2 => { + // A failed combine drops the held shares; holders resubmit. + let ark = bundle + .recover(&held[0], &held[1]) + .inspect_err(|_| intake.clear())?; + self.unlock_with(ark, &bundle).await?; + intake.clear(); + ShareOutcome::Unlocked + } + KeyringState::Unlocked if held.len() == 3 => { + let keys = self.keys()?; + let gen = keys + .gens + .get(&bundle.generation) + .ok_or(AtRestError::Inconsistent( + "escrowed generation is not loaded", + ))?; + let unlock = bundle + .test_unlock(&gen.ark, &held) + .inspect_err(|_| intake.clear())?; + let record = VerificationRecord::seal( + &gen.ark, + &bundle, + &unlock, + self.config.app_id.clone(), + self.config.compose_hash.clone(), + now_rfc3339(), + )?; + let json = serde_json::to_string(&record).expect("serializable"); + self.store + .save_record(&efp.to_string(), json.clone()) + .await + .map_err(AtRestError::from)?; + self.mirror + .put( + &format!("{MIRROR_PREFIX}record-{efp}.json"), + json.into_bytes(), + ) + .await + .map_err(mirror_err)?; + intake.clear(); + tracing::warn!(%efp, "at-rest escrow verified: all three shares rebuilt the ARK"); + ShareOutcome::Verified { efp } + } + _ => ShareOutcome::Held { + shares_held: intake.held_indices(), + }, + }; + Ok(outcome) + } + + /// Locked โ†’ unlocked with a recovered newest ARK: re-seal every + /// generation under this node's W, mirror, and serve keys. + async fn unlock_with(&self, ark: Ark, bundle: &EscrowBundle) -> Result<(), AtRestError> { + let rows = self.store.keys().await?; + let newest = rows.last().ok_or(AtRestError::NotProvisioned)?; + if newest.generation as u32 != bundle.generation || ark_id_of(newest)? != bundle.ark_id { + return Err(AtRestError::Inconsistent( + "recovered ARK is not the newest generation", + )); + } + let keys = keys_from_newest(&self.w, &rows, ark)?; + for (g, gen) in &keys.gens { + self.store + .reseal_key( + *g as i32, + W_LABEL, + seal_ark(&self.w, &gen.ark, &gen.ark_id, *g), + ) + .await?; + } + for row in self.store.keys().await? { + Self::mirror_row(&*self.mirror, &row).await?; + } + *self.phase.write() = Phase::Unlocked(Arc::new(keys)); + tracing::warn!( + generation = bundle.generation, + "at-rest keys recovered from escrow and resealed under this node's root" + ); + Ok(()) + } + + /// Start ARK generation n+1. Older generations stay readable through the + /// chain; new DEKs wrap under the new KEK. The new generation has no + /// verified escrow until a ceremony runs, so the gate closes. + pub async fn rotate_ark(&self) -> Result { + let keys = self.keys()?; + let (g, old) = keys.newest(); + let next = g + 1; + let ark = Ark::generate(); + let ark_id = ArkId::generate(); + self.store + .insert_key(at_rest_key::Model { + generation: next as i32, + ark_id: ark_id.0.to_vec(), + w_label: W_LABEL.into(), + sealed: seal_ark(&self.w, &ark, &ark_id, next), + chain: None, + created_at: now_rfc3339(), + }) + .await?; + self.store + .set_chain(g as i32, chain_ark(&ark, &ark_id, &old.ark, &old.ark_id, g)) + .await?; + let rows = self.store.keys().await?; + let keys = unlock_rows(&self.w, &rows)?.ok_or(AtRestError::Inconsistent( + "a just-written generation does not open", + ))?; + for row in &rows { + Self::mirror_row(&*self.mirror, row).await?; + } + *self.phase.write() = Phase::Unlocked(Arc::new(keys)); + self.current.lock().clear(); + self.ensure_escrow().await?; + Ok(next) + } + + async fn load_dek( + &self, + keys: &Keys, + row: &space_data_key::Model, + ) -> Result { + let dek_id = DekId::new(row.dek_id as u32); + let gen = keys.gens.get(&(row.generation as u32)).ok_or_else(|| { + backend(AtRestError::Inconsistent( + "data key names an unknown ARK generation", + )) + })?; + let key = unwrap_dek( + &gen.kek, + &row.wrapped, + &row.space, + dek_id, + row.generation as u32, + &gen.ark_id, + ) + .map_err(backend)?; + self.deks + .lock() + .put((row.space.clone(), dek_id.get()), key.clone()); + Ok(key) + } +} + +fn locked(e: AtRestError) -> DataKeyError { + match e { + AtRestError::Locked | AtRestError::NotProvisioned => DataKeyError::Locked, + other => backend(other), + } +} + +#[async_trait::async_trait] +impl DataKeyProvider for AtRestKeyring { + async fn current_key(&self, space: &str) -> Result { + let keys = self.keys().map_err(locked)?; + let cached = self.current.lock().get(space).copied(); + if let Some(id) = cached { + return self.key(space, id).await; + } + let row = match self.store.newest_dek(space).await.map_err(backend)? { + Some(row) => row, + None => { + let _create = self.dek_create.lock().await; + match self.store.newest_dek(space).await.map_err(backend)? { + Some(row) => row, + None => { + let (g, gen) = keys.newest(); + let id = DekId::new(1); + let (wrapped, _) = new_dek(&gen.kek, space, id, g, &gen.ark_id); + self.store + .insert_dek(space_data_key::Model { + space: space.to_owned(), + dek_id: 1, + generation: g as i32, + wrapped, + created_at: now_rfc3339(), + }) + .await + .map_err(backend)?; + // Another process may have won; serve whatever is stored. + self.store + .dek(space, 1) + .await + .map_err(backend)? + .ok_or_else(|| { + backend(AtRestError::Inconsistent("inserted data key is missing")) + })? + } + } + } + }; + let key = self.load_dek(&keys, &row).await?; + self.current.lock().put(space.to_owned(), key.id()); + Ok(key) + } + + async fn key(&self, space: &str, dek_id: DekId) -> Result { + let keys = self.keys().map_err(locked)?; + if let Some(key) = self.deks.lock().get(&(space.to_owned(), dek_id.get())) { + return Ok(key.clone()); + } + let Ok(id) = i32::try_from(dek_id.get()) else { + return Err(DataKeyError::UnknownKey { dek_id }); + }; + match self.store.dek(space, id).await.map_err(backend)? { + Some(row) => self.load_dek(&keys, &row).await, + None => Err(DataKeyError::UnknownKey { dek_id }), + } + } + + fn object_name_key(&self) -> Result { + Ok(self.keys().map_err(locked)?.nk.clone()) + } +} + +#[cfg(test)] +mod tests; diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs new file mode 100644 index 00000000..12adb14a --- /dev/null +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -0,0 +1,597 @@ +use std::io::Read; +use std::sync::Arc; + +use sea_orm::{ConnectOptions, Database, DatabaseConnection, EntityTrait}; +use sea_orm_migration::MigratorTrait; +use tinycloud_at_rest_escrow::{wrap_share, EscrowBundle, HolderRecipient, SharePayload}; + +use super::*; +use crate::at_rest::store::MemoryMirror; +use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; + +const ROOT: [u8; 64] = [7; 64]; +const NEW_ROOT: [u8; 64] = [9; 64]; + +async fn db() -> DatabaseConnection { + let mut options = ConnectOptions::new("sqlite::memory:".to_string()); + options.max_connections(1).sqlx_logging(false); + let db = Database::connect(options).await.unwrap(); + crate::migrations::Migrator::up(&db, None).await.unwrap(); + db +} + +struct Holders { + ids: Vec, + recipients: Vec, +} + +fn holders() -> Holders { + let ids: Vec<_> = (0..3).map(|_| age::x25519::Identity::generate()).collect(); + let recipients = ["sam", "hunter", "safe"] + .iter() + .zip(&ids) + .map(|(h, id)| HolderRecipient { + holder: h.to_string(), + recipient: id.to_public().to_string(), + }) + .collect(); + Holders { ids, recipients } +} + +impl Holders { + /// What holder `i` gets from `age -d`. + fn decrypt(&self, bundle: &EscrowBundle, i: usize) -> SharePayload { + let armored = &bundle.shares[i].ciphertext; + let decryptor = + age::Decryptor::new_buffered(age::armor::ArmoredReader::new(armored.as_bytes())) + .unwrap(); + let mut reader = decryptor + .decrypt(std::iter::once(&self.ids[i] as &dyn age::Identity)) + .unwrap(); + let mut plain = Vec::new(); + reader.read_to_end(&mut plain).unwrap(); + bundle.open_share(&plain).unwrap() + } +} + +fn config(h: &Holders, pinned: Option) -> KeyringConfig { + KeyringConfig { + provision: true, + holders: h.recipients.clone(), + recipient_policy: RecipientPolicy::AllowSoftware, + pinned_efp: pinned, + app_id: Some("app".into()), + compose_hash: Some("compose".into()), + } +} + +async fn open( + db: &DatabaseConnection, + mirror: &Arc, + root: &[u8], + config: KeyringConfig, +) -> Result { + AtRestKeyring::open(AtRestStore::new(db.clone()), mirror.clone(), root, config).await +} + +/// Holder `i` wraps their share to the keyring's current recovery key. +async fn submit(k: &AtRestKeyring, h: &Holders, i: usize) -> Result { + let view = k.escrow_view().await.unwrap().unwrap(); + let share = h.decrypt(&view.bundle, i); + let wrapped = wrap_share(&view.recovery_recipient, &share).unwrap(); + k.submit_share(wrapped.as_bytes()).await +} + +#[tokio::test] +async fn provision_then_reopen_with_same_root_serves_the_same_keys() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + let dek = k.current_key("space-a").await.unwrap(); + assert_eq!(dek.id(), DekId::new(1)); + let nk = k.object_name_key().unwrap(); + // Mirror holds the sealed ARK and the escrow bundle. + let names = mirror.names(); + assert!(names.iter().any(|n| n.starts_with(ARK_MIRROR_PREFIX))); + assert!(names.iter().any(|n| n.contains("/escrow-"))); + drop(k); + + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + assert_eq!(k.current_key("space-a").await.unwrap().bytes(), dek.bytes()); + assert_eq!(k.object_name_key().unwrap().bytes(), nk.bytes()); + assert_eq!(at_rest_key::Entity::find().all(&db).await.unwrap().len(), 1); +} + +#[tokio::test] +async fn nothing_is_created_without_provisioning() { + let (db, mirror) = (db().await, Arc::new(MemoryMirror::default())); + let k = open(&db, &mirror, &ROOT, KeyringConfig::default()) + .await + .unwrap(); + assert_eq!(k.state(), KeyringState::NotProvisioned); + assert!(matches!( + k.current_key("s").await, + Err(DataKeyError::Locked) + )); + assert!(at_rest_key::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); + assert!(mirror.names().is_empty()); +} + +#[tokio::test] +async fn data_keys_converge_are_bound_and_never_invented() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = Arc::new(open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap()); + // A second keyring on the same database stands in for another process. + let other = Arc::new(open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap()); + let tasks: Vec<_> = (0..8) + .map(|i| { + let k = if i % 2 == 0 { k.clone() } else { other.clone() }; + tokio::spawn(async move { k.current_key("space-a").await.unwrap() }) + }) + .collect(); + let keys: Vec = futures::future::join_all(tasks) + .await + .into_iter() + .map(|r| r.unwrap()) + .collect(); + assert!(keys + .iter() + .all(|key| key.id() == DekId::new(1) && key.bytes() == keys[0].bytes())); + assert_eq!( + space_data_key::Entity::find().all(&db).await.unwrap().len(), + 1 + ); + + let b = k.current_key("space-b").await.unwrap(); + assert_ne!(b.bytes(), keys[0].bytes()); + assert!(matches!( + k.key("space-a", DekId::new(2)).await, + Err(DataKeyError::UnknownKey { dek_id }) if dek_id == DekId::new(2) + )); + assert!(matches!( + k.key("space-a", DekId::new(u32::MAX)).await, + Err(DataKeyError::UnknownKey { .. }) + )); + + // A wrapped DEK copied to another space fails authentication instead of + // serving space-a's key for space-c. + let row = space_data_key::Entity::find_by_id(("space-a".to_string(), 1)) + .one(&db) + .await + .unwrap() + .unwrap(); + AtRestStore::new(db.clone()) + .insert_dek(space_data_key::Model { + space: "space-c".into(), + ..row + }) + .await + .unwrap(); + let fresh = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert!(matches!( + fresh.key("space-c", DekId::new(1)).await, + Err(DataKeyError::Backend(_)) + )); +} + +#[tokio::test] +async fn new_root_boots_locked_and_two_shares_unlock_without_restart() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let dek = k.current_key("space-a").await.unwrap(); + let nk = k.object_name_key().unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + + // New app id / KMS: a different root. Locked, not lost. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert_eq!(k.state(), KeyringState::Locked); + assert!(matches!( + k.current_key("space-a").await, + Err(DataKeyError::Locked) + )); + assert!(matches!(k.object_name_key(), Err(DataKeyError::Locked))); + assert_eq!(k.gate(false).await, Err(GateRefusal::Locked)); + + // One share does nothing. + assert_eq!( + submit(&k, &h, 2).await.unwrap(), + ShareOutcome::Held { + shares_held: vec![3] + } + ); + assert_eq!(k.state(), KeyringState::Locked); + // A second distinct share recovers and reseals. + assert_eq!(submit(&k, &h, 0).await.unwrap(), ShareOutcome::Unlocked); + assert_eq!(k.state(), KeyringState::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + dek.bytes() + ); + assert_eq!(k.object_name_key().unwrap().bytes(), nk.bytes()); + drop(k); + + // Resealed under the new root: it now boots unlocked; the old root is locked out. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + assert_eq!(k.current_key("space-a").await.unwrap().bytes(), dek.bytes()); + let old = open(&db, &mirror, &ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert_eq!(old.state(), KeyringState::Locked); +} + +#[tokio::test] +async fn locked_node_trusts_only_the_pinned_bundle() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let real = k.escrow_view().await.unwrap().unwrap(); + drop(k); + + // No pin: nothing to check shares against. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, None)) + .await + .unwrap(); + assert!(k.escrow_view().await.unwrap().is_none()); + let share = h.decrypt(&real.bundle, 0); + let wrapped = wrap_share(&k.intake.lock().await.identity.recipient(), &share).unwrap(); + assert!(matches!( + k.submit_share(wrapped.as_bytes()).await, + Err(ShareRefusal::NoBundle(_)) + )); + + // A pin that isn't the stored bundle (e.g. storage swapped the bundle). + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(Digest32([1; 32])))) + .await + .unwrap(); + assert!(k.escrow_view().await.unwrap().is_none()); + + // Shares of a different ARK (another bundle) are refused before a slot. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(real.efp))) + .await + .unwrap(); + let foreign = EscrowBundle::create( + &Ark::generate(), + ArkId::generate(), + 1, + &h.recipients, + RecipientPolicy::AllowSoftware, + ) + .unwrap(); + let wrapped = wrap_share( + &k.escrow_view().await.unwrap().unwrap().recovery_recipient, + &h.decrypt(&foreign, 1), + ) + .unwrap(); + assert!(matches!( + k.submit_share(wrapped.as_bytes()).await, + Err(ShareRefusal::Invalid(_)) + )); + assert!(k + .escrow_view() + .await + .unwrap() + .unwrap() + .shares_held + .is_empty()); + // Junk and wrong-key age files too. + assert!(k.submit_share(b"junk").await.is_err()); +} + +#[tokio::test] +async fn pitr_restore_without_ark_rows_recovers_from_the_mirror() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let dek = k.current_key("space-a").await.unwrap(); + drop(k); + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + dek.bytes() + ); +} + +#[tokio::test] +async fn never_provisions_over_uncertainty() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + k.current_key("space-a").await.unwrap(); + drop(k); + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + + // Mirror unreachable: fail, don't make a fresh ARK. + mirror.set_failing(true); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::Mirror(_)) + )); + // Mirror empty but data keys exist: inconsistent, don't make a fresh ARK. + mirror.set_failing(false); + mirror.clear(); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::Inconsistent(_)) + )); + assert!(at_rest_key::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn gate_requires_pinned_verified_escrow_for_the_configured_recipients() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + assert_eq!( + k.gate(false).await, + Err(GateRefusal::NotPinned { + efp: efp.to_string() + }) + ); + drop(k); + + let k = open(&db, &mirror, &ROOT, config(&h, Some(Digest32([2; 32])))) + .await + .unwrap(); + assert!(matches!( + k.gate(false).await, + Err(GateRefusal::PinMismatch { .. }) + )); + drop(k); + + // Pinned but no ceremony yet. + let k = open(&db, &mirror, &ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert_eq!( + k.gate(false).await, + Err(GateRefusal::NotVerified { + efp: efp.to_string() + }) + ); + + // Ceremony test-unlock: two shares are held, the third verifies. + assert_eq!( + submit(&k, &h, 0).await.unwrap(), + ShareOutcome::Held { + shares_held: vec![1] + } + ); + assert_eq!( + submit(&k, &h, 2).await.unwrap(), + ShareOutcome::Held { + shares_held: vec![1, 3] + } + ); + assert_eq!( + submit(&k, &h, 1).await.unwrap(), + ShareOutcome::Verified { efp } + ); + assert_eq!(k.gate(false).await, Ok(GatePass::Verified { efp })); + assert!(k.escrow_view().await.unwrap().unwrap().verified); + let record = at_rest_escrow::Entity::find_by_id(efp.to_string()) + .one(&db) + .await + .unwrap() + .unwrap() + .record + .unwrap(); + let parsed: VerificationRecord = serde_json::from_str(&record).unwrap(); + assert_eq!(parsed.app_id.as_deref(), Some("app")); + drop(k); + + // A record edited in storage no longer verifies. + let mut forged: serde_json::Value = serde_json::from_str(&record).unwrap(); + forged["verified_at"] = serde_json::json!("2030-01-01T00:00:00Z"); + AtRestStore::new(db.clone()) + .save_record(&efp.to_string(), forged.to_string()) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert!(matches!( + k.gate(false).await, + Err(GateRefusal::NotVerified { .. }) + )); + AtRestStore::new(db.clone()) + .save_record(&efp.to_string(), record) + .await + .unwrap(); + assert_eq!(k.gate(false).await, Ok(GatePass::Verified { efp })); + drop(k); + + // New recipients in config re-split the same ARK: a new bundle, and the + // old pin no longer opens the gate. + let changed = holders(); + let k = open(&db, &mirror, &ROOT, config(&changed, Some(efp))) + .await + .unwrap(); + let new_efp = k.escrow_view().await.unwrap().unwrap().efp; + assert_ne!(new_efp, efp); + assert!(matches!( + k.gate(false).await, + Err(GateRefusal::PinMismatch { .. }) + )); + let bundles = at_rest_escrow::Entity::find().all(&db).await.unwrap(); + assert_eq!(bundles.len(), 2); + assert_eq!(bundles.iter().filter(|b| b.active).count(), 1); +} + +#[tokio::test] +async fn software_holders_need_the_opt_in_to_escrow_and_to_pass_the_gate() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let hardware_only = |pinned| KeyringConfig { + recipient_policy: RecipientPolicy::HardwareOnly, + ..config(&h, pinned) + }; + // Hardware-only config with X25519 holders: no escrow is made. + assert!(matches!( + open(&db, &mirror, &ROOT, hardware_only(None)).await, + Err(AtRestError::Escrow(_)) + )); + // Escrowed and verified under the opt-in... + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + let k = open(&db, &mirror, &ROOT, config(&h, Some(efp))) + .await + .unwrap(); + for i in 0..3 { + submit(&k, &h, i).await.unwrap(); + } + assert_eq!(k.gate(false).await, Ok(GatePass::Verified { efp })); + drop(k); + // ...still doesn't open the gate once the opt-in is removed. + let k = open(&db, &mirror, &ROOT, hardware_only(Some(efp))) + .await + .unwrap(); + assert_eq!(k.gate(false).await, Err(GateRefusal::SoftwareRecipients)); +} + +#[tokio::test] +async fn self_host_without_escrow_needs_the_explicit_acknowledgement() { + let (db, mirror) = (db().await, Arc::new(MemoryMirror::default())); + let config = KeyringConfig { + provision: true, + ..Default::default() + }; + let k = open(&db, &mirror, &ROOT, config).await.unwrap(); + assert_eq!(k.gate(false).await, Err(GateRefusal::NoEscrow)); + assert_eq!(k.gate(true).await, Ok(GatePass::Unescrowed)); + assert!(k.escrow_view().await.unwrap().is_none()); + assert!(at_rest_escrow::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn rotation_keeps_old_generations_and_survives_recovery() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let old = k.current_key("space-a").await.unwrap(); + let nk = k.object_name_key().unwrap(); + assert_eq!(k.rotate_ark().await.unwrap(), 2); + // Old DEK still served; a new space wraps under generation 2; NK pinned to gen 1. + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + old.bytes() + ); + k.current_key("space-b").await.unwrap(); + let b_row = space_data_key::Entity::find_by_id(("space-b".to_string(), 1)) + .one(&db) + .await + .unwrap() + .unwrap(); + assert_eq!(b_row.generation, 2); + assert_eq!(k.object_name_key().unwrap().bytes(), nk.bytes()); + let view = k.escrow_view().await.unwrap().unwrap(); + assert_eq!(view.generation, 2); + assert!(!view.verified); + drop(k); + + // Escrow of generation 2 alone recovers both generations on a new root. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(view.efp))) + .await + .unwrap(); + assert_eq!(k.state(), KeyringState::Locked); + submit(&k, &h, 0).await.unwrap(); + assert_eq!(submit(&k, &h, 1).await.unwrap(), ShareOutcome::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + old.bytes() + ); + assert_eq!(k.object_name_key().unwrap().bytes(), nk.bytes()); + assert_eq!(at_rest_key::Entity::find().all(&db).await.unwrap().len(), 2); +} + +#[tokio::test] +async fn share_intake_is_rate_limited_per_boot() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + for _ in 0..SHARE_BURST { + assert!(!matches!( + k.submit_share(b"junk").await, + Err(ShareRefusal::RateLimited) + )); + } + assert!(matches!( + k.submit_share(b"junk").await, + Err(ShareRefusal::RateLimited) + )); + // Valid shares are throttled the same way. + assert!(matches!( + submit(&k, &h, 0).await, + Err(ShareRefusal::RateLimited) + )); +} + +#[tokio::test] +async fn postgres_two_nodes_converge_on_one_ark_and_one_dek_per_space() { + let Some(pg) = crate::test_support::PostgresTestSchema::new( + "postgres_two_nodes_converge_on_one_ark_and_one_dek_per_space", + ) + .await + else { + return; + }; + let (mirror, h) = (Arc::new(MemoryMirror::default()), holders()); + // Two processes booting at once against the same database (pool of 4). + let (a, b) = tokio::join!( + open(&pg.conn, &mirror, &ROOT, config(&h, None)), + open(&pg.conn, &mirror, &ROOT, config(&h, None)), + ); + let (a, b) = (Arc::new(a.unwrap()), Arc::new(b.unwrap())); + assert_eq!( + at_rest_key::Entity::find() + .all(&pg.conn) + .await + .unwrap() + .len(), + 1 + ); + assert_eq!( + a.object_name_key().unwrap().bytes(), + b.object_name_key().unwrap().bytes() + ); + + let tasks: Vec<_> = (0..16) + .map(|i| { + let k = if i % 2 == 0 { a.clone() } else { b.clone() }; + tokio::spawn(async move { k.current_key(&format!("space-{}", i % 4)).await.unwrap() }) + }) + .collect(); + let keys: Vec = futures::future::join_all(tasks) + .await + .into_iter() + .map(|r| r.unwrap()) + .collect(); + for (i, key) in keys.iter().enumerate() { + assert_eq!(key.id(), DekId::new(1)); + assert_eq!(key.bytes(), keys[i % 4].bytes(), "space-{} diverged", i % 4); + } + assert_eq!( + space_data_key::Entity::find() + .all(&pg.conn) + .await + .unwrap() + .len(), + 4 + ); +} diff --git a/tinycloud-core/src/at_rest/mod.rs b/tinycloud-core/src/at_rest/mod.rs index d63ad1fd..27914be0 100644 --- a/tinycloud-core/src/at_rest/mod.rs +++ b/tinycloud-core/src/at_rest/mod.rs @@ -2,9 +2,12 @@ //! is switched off until the key-continuity gate exists. mod envelope; +mod hierarchy; mod key; +pub mod keyring; mod names; mod provider; +pub mod store; mod stream; #[cfg(any(test, feature = "test-util"))] mod testing; @@ -21,3 +24,6 @@ pub use provider::{DataKeyError, DataKeyProvider}; pub use stream::{DecryptReader, EncryptReader, SegmentRange}; #[cfg(any(test, feature = "test-util"))] pub use testing::FixedKeyProvider; + +pub use keyring::{AtRestKeyring, KeyringConfig, KeyringState}; +pub use store::{AtRestStore, KeyMirror}; diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs new file mode 100644 index 00000000..dfd2b8a4 --- /dev/null +++ b/tinycloud-core/src/at_rest/store.rs @@ -0,0 +1,335 @@ +//! Persistence for the at-rest keys: PG/SQLite rows plus a mirror of the +//! sealed ARK rows and escrow bundles in the block store (TC-887 plan ยง5). +//! +//! The mirror covers a database restored to a point before an ARK row was +//! written: sealed blobs in object storage outlive such a restore. + +use std::sync::Arc; + +use base64::Engine; +use sea_orm::{ + sea_query::OnConflict, ActiveValue::Set, ColumnTrait, DbErr, EntityTrait, PaginatorTrait, + QueryFilter, QueryOrder, TransactionTrait, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; + +use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; +use crate::RecoveringConnection; + +pub type MirrorError = Box; + +/// Named control objects next to the node's blocks (`_tc/at-rest/โ€ฆ`). The S3 +/// and filesystem block stores implement it in the server crate. +#[async_trait::async_trait] +pub trait KeyMirror: Send + Sync { + async fn put(&self, name: &str, bytes: Vec) -> Result<(), MirrorError>; + async fn get(&self, name: &str) -> Result>, MirrorError>; + /// Names under `prefix`, in any order. + async fn list(&self, prefix: &str) -> Result, MirrorError>; +} + +pub const MIRROR_PREFIX: &str = "_tc/at-rest/"; +pub(crate) const ARK_MIRROR_PREFIX: &str = "_tc/at-rest/ark-"; + +pub(crate) fn ark_mirror_name(generation: u32, ark_id_hex: &str) -> String { + format!("{ARK_MIRROR_PREFIX}{generation:010}-{ark_id_hex}.json") +} + +pub(crate) fn escrow_mirror_name(efp: &str) -> String { + format!("{MIRROR_PREFIX}escrow-{efp}.json") +} + +/// The mirrored form of an `at_rest_key` row. +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct MirroredArk { + pub generation: u32, + pub ark_id: String, + pub w_label: String, + pub sealed: String, + pub chain: Option, + pub created_at: String, +} + +fn b64() -> base64::engine::GeneralPurpose { + base64::engine::general_purpose::STANDARD +} + +impl MirroredArk { + pub fn from_row(row: &at_rest_key::Model) -> Self { + Self { + generation: row.generation as u32, + ark_id: hex::encode(&row.ark_id), + w_label: row.w_label.clone(), + sealed: b64().encode(&row.sealed), + chain: row.chain.as_ref().map(|c| b64().encode(c)), + created_at: row.created_at.clone(), + } + } + + pub fn into_row(self) -> Result { + Ok(at_rest_key::Model { + generation: i32::try_from(self.generation)?, + ark_id: hex::decode(&self.ark_id)?, + w_label: self.w_label, + sealed: b64().decode(&self.sealed)?, + chain: self.chain.map(|c| b64().decode(c)).transpose()?, + created_at: self.created_at, + }) + } +} + +/// Row access. On SQLite, writes take the node's shared writer lock. +#[derive(Clone)] +pub struct AtRestStore { + db: RecoveringConnection, + sqlite_writer_lock: Option>>, +} + +fn ignore_conflict(r: Result) -> Result<(), DbErr> { + match r { + Ok(_) | Err(DbErr::RecordNotInserted) => Ok(()), + Err(e) => Err(e), + } +} + +impl AtRestStore { + pub fn new(db: impl Into) -> Self { + Self { + db: db.into(), + sqlite_writer_lock: None, + } + } + + pub fn with_sqlite_writer_lock(mut self, lock: Option>>) -> Self { + self.sqlite_writer_lock = lock; + self + } + + async fn writer(&self) -> Option> { + match &self.sqlite_writer_lock { + Some(lock) => Some(lock.lock().await), + None => None, + } + } + + pub async fn keys(&self) -> Result, DbErr> { + at_rest_key::Entity::find() + .order_by_asc(at_rest_key::Column::Generation) + .all(&self.db) + .await + } + + /// Insert unless the generation exists. The caller re-reads. + pub async fn insert_key(&self, row: at_rest_key::Model) -> Result<(), DbErr> { + let _w = self.writer().await; + ignore_conflict( + at_rest_key::Entity::insert(at_rest_key::ActiveModel { + generation: Set(row.generation), + ark_id: Set(row.ark_id), + w_label: Set(row.w_label), + sealed: Set(row.sealed), + chain: Set(row.chain), + created_at: Set(row.created_at), + }) + .on_conflict( + OnConflict::column(at_rest_key::Column::Generation) + .do_nothing() + .to_owned(), + ) + .exec(&self.db) + .await, + ) + } + + /// Replace a generation's seal (after recovery under a new W). + pub async fn reseal_key( + &self, + generation: i32, + w_label: &str, + sealed: Vec, + ) -> Result<(), DbErr> { + let _w = self.writer().await; + at_rest_key::Entity::update_many() + .col_expr(at_rest_key::Column::WLabel, w_label.into()) + .col_expr(at_rest_key::Column::Sealed, sealed.into()) + .filter(at_rest_key::Column::Generation.eq(generation)) + .exec(&self.db) + .await + .map(|_| ()) + } + + pub async fn set_chain(&self, generation: i32, chain: Vec) -> Result<(), DbErr> { + let _w = self.writer().await; + at_rest_key::Entity::update_many() + .col_expr(at_rest_key::Column::Chain, chain.into()) + .filter(at_rest_key::Column::Generation.eq(generation)) + .exec(&self.db) + .await + .map(|_| ()) + } + + pub async fn has_space_keys(&self) -> Result { + Ok(space_data_key::Entity::find().count(&self.db).await? > 0) + } + + pub async fn active_escrow( + &self, + generation: i32, + ) -> Result, DbErr> { + at_rest_escrow::Entity::find() + .filter(at_rest_escrow::Column::Generation.eq(generation)) + .filter(at_rest_escrow::Column::Active.eq(true)) + .one(&self.db) + .await + } + + pub async fn escrow(&self, efp: &str) -> Result, DbErr> { + at_rest_escrow::Entity::find_by_id(efp.to_owned()) + .one(&self.db) + .await + } + + /// Make `row` the generation's only active bundle. Older bundles stay. + pub async fn activate_escrow(&self, row: at_rest_escrow::Model) -> Result<(), DbErr> { + let _w = self.writer().await; + let tx = self.db.begin().await?; + at_rest_escrow::Entity::update_many() + .col_expr(at_rest_escrow::Column::Active, false.into()) + .filter(at_rest_escrow::Column::Generation.eq(row.generation)) + .exec(&tx) + .await?; + ignore_conflict( + at_rest_escrow::Entity::insert(at_rest_escrow::ActiveModel { + efp: Set(row.efp.clone()), + generation: Set(row.generation), + bundle: Set(row.bundle), + active: Set(true), + record: Set(row.record), + created_at: Set(row.created_at), + }) + .on_conflict( + OnConflict::column(at_rest_escrow::Column::Efp) + .do_nothing() + .to_owned(), + ) + .exec(&tx) + .await, + )?; + at_rest_escrow::Entity::update_many() + .col_expr(at_rest_escrow::Column::Active, true.into()) + .filter(at_rest_escrow::Column::Efp.eq(row.efp)) + .exec(&tx) + .await?; + tx.commit().await + } + + pub async fn save_record(&self, efp: &str, record_json: String) -> Result<(), DbErr> { + let _w = self.writer().await; + at_rest_escrow::Entity::update_many() + .col_expr(at_rest_escrow::Column::Record, record_json.into()) + .filter(at_rest_escrow::Column::Efp.eq(efp)) + .exec(&self.db) + .await + .map(|_| ()) + } + + pub async fn newest_dek(&self, space: &str) -> Result, DbErr> { + space_data_key::Entity::find() + .filter(space_data_key::Column::Space.eq(space)) + .order_by_desc(space_data_key::Column::DekId) + .one(&self.db) + .await + } + + pub async fn dek( + &self, + space: &str, + dek_id: i32, + ) -> Result, DbErr> { + space_data_key::Entity::find_by_id((space.to_owned(), dek_id)) + .one(&self.db) + .await + } + + /// Insert unless `(space, dek_id)` exists. The caller re-reads, so + /// concurrent creators converge on the first row written. + pub async fn insert_dek(&self, row: space_data_key::Model) -> Result<(), DbErr> { + let _w = self.writer().await; + ignore_conflict( + space_data_key::Entity::insert(space_data_key::ActiveModel { + space: Set(row.space), + dek_id: Set(row.dek_id), + generation: Set(row.generation), + wrapped: Set(row.wrapped), + created_at: Set(row.created_at), + }) + .on_conflict( + OnConflict::columns([space_data_key::Column::Space, space_data_key::Column::DekId]) + .do_nothing() + .to_owned(), + ) + .exec(&self.db) + .await, + ) + } +} + +/// In-memory mirror for tests. +#[cfg(any(test, feature = "test-util"))] +#[derive(Default)] +pub struct MemoryMirror { + objects: parking_lot::Mutex>>, + fail: std::sync::atomic::AtomicBool, +} + +#[cfg(any(test, feature = "test-util"))] +impl MemoryMirror { + /// Make every call fail, like an unreachable bucket. + pub fn set_failing(&self, fail: bool) { + self.fail.store(fail, std::sync::atomic::Ordering::SeqCst); + } + + pub fn names(&self) -> Vec { + self.objects.lock().keys().cloned().collect() + } + + pub fn clear(&self) { + self.objects.lock().clear(); + } + + fn check(&self) -> Result<(), MirrorError> { + if self.fail.load(std::sync::atomic::Ordering::SeqCst) { + Err("mirror unavailable".into()) + } else { + Ok(()) + } + } +} + +#[cfg(any(test, feature = "test-util"))] +#[async_trait::async_trait] +impl KeyMirror for MemoryMirror { + async fn put(&self, name: &str, bytes: Vec) -> Result<(), MirrorError> { + self.check()?; + self.objects.lock().insert(name.to_owned(), bytes); + Ok(()) + } + + async fn get(&self, name: &str) -> Result>, MirrorError> { + self.check()?; + Ok(self.objects.lock().get(name).cloned()) + } + + async fn list(&self, prefix: &str) -> Result, MirrorError> { + self.check()?; + Ok(self + .objects + .lock() + .keys() + .filter(|k| k.starts_with(prefix)) + .cloned() + .collect()) + } +} diff --git a/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs b/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs new file mode 100644 index 00000000..219afcbb --- /dev/null +++ b/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs @@ -0,0 +1,131 @@ +//! TC-887 S1: tables for the at-rest key hierarchy and its escrow. Empty +//! until a node is configured to provision at-rest keys. + +use sea_orm_migration::prelude::*; + +use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(at_rest_key::Entity) + .if_not_exists() + .col( + ColumnDef::new(at_rest_key::Column::Generation) + .integer() + .not_null() + .primary_key(), + ) + .col(ColumnDef::new(at_rest_key::Column::ArkId).blob().not_null()) + .col( + ColumnDef::new(at_rest_key::Column::WLabel) + .string() + .not_null(), + ) + .col( + ColumnDef::new(at_rest_key::Column::Sealed) + .blob() + .not_null(), + ) + .col(ColumnDef::new(at_rest_key::Column::Chain).blob()) + .col( + ColumnDef::new(at_rest_key::Column::CreatedAt) + .string() + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(at_rest_escrow::Entity) + .if_not_exists() + .col( + ColumnDef::new(at_rest_escrow::Column::Efp) + .string() + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(at_rest_escrow::Column::Generation) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(at_rest_escrow::Column::Bundle) + .text() + .not_null(), + ) + .col( + ColumnDef::new(at_rest_escrow::Column::Active) + .boolean() + .not_null(), + ) + .col(ColumnDef::new(at_rest_escrow::Column::Record).text()) + .col( + ColumnDef::new(at_rest_escrow::Column::CreatedAt) + .string() + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(space_data_key::Entity) + .if_not_exists() + .col( + ColumnDef::new(space_data_key::Column::Space) + .string() + .not_null(), + ) + .col( + ColumnDef::new(space_data_key::Column::DekId) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(space_data_key::Column::Generation) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(space_data_key::Column::Wrapped) + .blob() + .not_null(), + ) + .col( + ColumnDef::new(space_data_key::Column::CreatedAt) + .string() + .not_null(), + ) + .primary_key( + Index::create() + .col(space_data_key::Column::Space) + .col(space_data_key::Column::DekId), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(space_data_key::Entity).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(at_rest_escrow::Entity).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(at_rest_key::Entity).to_owned()) + .await + } +} diff --git a/tinycloud-core/src/migrations/mod.rs b/tinycloud-core/src/migrations/mod.rs index e8f7c992..6adf6c9a 100644 --- a/tinycloud-core/src/migrations/mod.rs +++ b/tinycloud-core/src/migrations/mod.rs @@ -26,6 +26,7 @@ pub mod m20261007_000000_database_alias; pub mod m20261007_010000_database_identity_fence; pub mod m20261014_000000_delegation_cid_alias; pub mod m20261015_000000_delegation_cursor_index; +pub mod m20261017_000000_at_rest_keys; pub struct Migrator; @@ -60,6 +61,7 @@ impl MigratorTrait for Migrator { Box::new(m20261007_010000_database_identity_fence::Migration), Box::new(m20261014_000000_delegation_cid_alias::Migration), Box::new(m20261015_000000_delegation_cursor_index::Migration), + Box::new(m20261017_000000_at_rest_keys::Migration), ] } } diff --git a/tinycloud-core/src/models/at_rest_escrow.rs b/tinycloud-core/src/models/at_rest_escrow.rs new file mode 100644 index 00000000..f2a38ae8 --- /dev/null +++ b/tinycloud-core/src/models/at_rest_escrow.rs @@ -0,0 +1,26 @@ +//! Escrow bundles of ARK generations and their ceremony verification +//! records (TC-887). Superseded bundles are kept, marked inactive. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "at_rest_escrow")] +pub struct Model { + /// Escrow fingerprint, lowercase hex. + #[sea_orm(primary_key, auto_increment = false)] + pub efp: String, + pub generation: i32, + /// `tinycloud_at_rest_escrow::EscrowBundle` as JSON. + pub bundle: String, + /// The bundle the node escrows to now (one per generation). + pub active: bool, + /// `tinycloud_at_rest_escrow::VerificationRecord` as JSON, after a + /// successful ceremony test-unlock. + pub record: Option, + pub created_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/tinycloud-core/src/models/at_rest_key.rs b/tinycloud-core/src/models/at_rest_key.rs new file mode 100644 index 00000000..e3dcf7f8 --- /dev/null +++ b/tinycloud-core/src/models/at_rest_key.rs @@ -0,0 +1,26 @@ +//! One generation of the at-rest root key (ARK), sealed under the node's +//! wrap key W. Rows are never deleted (TC-887). + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "at_rest_key")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub generation: i32, + /// 16 random bytes. + pub ark_id: Vec, + /// Which W sealed `sealed` (the HKDF info label). + pub w_label: String, + /// `nonce(12) โ€– AES-256-GCM(W, ARK)`. + pub sealed: Vec, + /// This generation's ARK sealed under the next generation's chain key; + /// set when the next generation is made. + pub chain: Option>, + pub created_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/tinycloud-core/src/models/mod.rs b/tinycloud-core/src/models/mod.rs index afefa687..b758657b 100644 --- a/tinycloud-core/src/models/mod.rs +++ b/tinycloud-core/src/models/mod.rs @@ -1,5 +1,7 @@ pub mod abilities; pub mod actor; +pub mod at_rest_escrow; +pub mod at_rest_key; pub mod current_kv; pub mod database_alias; pub mod database_artifact; @@ -40,3 +42,4 @@ pub mod share_policy_presentation_jti; pub mod share_session_handle; pub mod signed_kv_ticket; pub mod space; +pub mod space_data_key; diff --git a/tinycloud-core/src/models/space_data_key.rs b/tinycloud-core/src/models/space_data_key.rs new file mode 100644 index 00000000..fe28cdd8 --- /dev/null +++ b/tinycloud-core/src/models/space_data_key.rs @@ -0,0 +1,24 @@ +//! Per-space data keys (DEKs), wrapped under an ARK generation's KEK +//! (TC-887). Deleting a space's rows crypto-shreds its sealed blobs. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "space_data_key")] +pub struct Model { + /// Canonical `SpaceId::to_string()`. + #[sea_orm(primary_key, auto_increment = false)] + pub space: String, + #[sea_orm(primary_key, auto_increment = false)] + pub dek_id: i32, + /// The ARK generation whose KEK wrapped `wrapped`. + pub generation: i32, + /// `nonce(12) โ€– AES-256-GCM(KEK_g, DEK)`. + pub wrapped: Vec, + pub created_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} From 5fab9c09fefbd92c09b344a9dc915d5eb98967a7 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 07:19:58 +0000 Subject: [PATCH 02/15] feat(TC-887): expose the verified escrow record --- tinycloud-core/src/at_rest/keyring.rs | 12 ++++++++++++ tinycloud-core/src/at_rest/keyring/tests.rs | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index c05848da..c27063aa 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -592,6 +592,18 @@ impl AtRestKeyring { })) } + /// The active escrow's verification record, only if its MAC verifies + /// under the loaded ARK. `None` when locked, unescrowed or unverified. + pub async fn verification_record(&self) -> Result, AtRestError> { + if self.state() != KeyringState::Unlocked { + return Ok(None); + } + match self.ensure_escrow().await? { + Some(bundle) => self.verified_record(&bundle).await, + None => Ok(None), + } + } + async fn verified_record( &self, bundle: &EscrowBundle, diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index 12adb14a..71bdcf8c 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -383,6 +383,10 @@ async fn gate_requires_pinned_verified_escrow_for_the_configured_recipients() { ); assert_eq!(k.gate(false).await, Ok(GatePass::Verified { efp })); assert!(k.escrow_view().await.unwrap().unwrap().verified); + assert_eq!( + k.verification_record().await.unwrap().map(|r| r.efp), + Some(efp) + ); let record = at_rest_escrow::Entity::find_by_id(efp.to_string()) .one(&db) .await From 9ccd989814050371defb1ee02f52bb877e3d2679 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 07:58:42 +0000 Subject: [PATCH 03/15] fix(TC-887): atomic rotation, chain-only recovery, DEK mirror; review round 2 (#326) - Rotation writes generation n+1 and generation n's chain in one transaction, with the chain only on a NULL chain; a node that lost the race writes nothing, reloads, and gets AtRestError::Conflict. - Every older generation must open by its chain. At boot under the same root a missing or foreign chain is rebuilt from W and logged; under a new root (recovery) it is an error. Mirrored generations or chains the database lacks are restored at boot. unlock_with checks 1..n (no panic). - One active escrow bundle per generation: activate_escrow locks the generation's key row (FOR UPDATE), plus a partial unique index on PG and SQLite; active_escrow orders deterministically. - DEK rows are mirrored (named by a keyed tag, not the space) before a key is served, restored on a database miss, and a different mirrored DEK with the same id is refused rather than overwritten. - DataKey/ObjectNameKey::from_zeroizing (pub(crate)); the hierarchy no longer copies DEK/NK bytes out of zeroizing buffers. - GateRefusal::Unavailable for store/mirror/parse errors; stored bundles and records go through the versioned parsers; mirror objects carry a version. - Provisioning is refused over escrow rows or any _tc/at-rest/ object; the provisioned log names only the winning ark_id. - Migration down refuses while key rows exist. - Deterministic store-conflict tests (SQLite and PG), multi-thread PG test with concurrent re-splits; excluded test lockfiles refreshed. --- tinycloud-core/src/at_rest/hierarchy.rs | 24 +- tinycloud-core/src/at_rest/key.rs | 10 + tinycloud-core/src/at_rest/keyring.rs | 420 +++++++++++++----- tinycloud-core/src/at_rest/keyring/tests.rs | 351 ++++++++++++++- tinycloud-core/src/at_rest/store.rs | 136 +++++- .../m20261017_000000_at_rest_keys.rs | 31 ++ 6 files changed, 860 insertions(+), 112 deletions(-) diff --git a/tinycloud-core/src/at_rest/hierarchy.rs b/tinycloud-core/src/at_rest/hierarchy.rs index 83d74506..4854d71a 100644 --- a/tinycloud-core/src/at_rest/hierarchy.rs +++ b/tinycloud-core/src/at_rest/hierarchy.rs @@ -7,6 +7,8 @@ //! KEK_g = HKDF-SHA256(ARK_g, info "tinycloud/at-rest/kek/v1") //! NK = HKDF-SHA256(ARK_1, info "tinycloud/at-rest/object-names/v1") //! DEK = 32 random bytes per (space, dek_id): nonce โ€– AES-256-GCM(KEK_g, DEK, AAD dek) +//! TAG = HKDF-SHA256(ARK_1, info "tinycloud/at-rest/dek-mirror-tag/v1") +//! names a space's mirrored DEKs: hex(HKDF(TAG, info space)[..16]) //! ``` //! //! Every AAD starts with a length-prefixed domain string; variable fields are @@ -30,6 +32,7 @@ use super::{DataKey, DekId, ObjectNameKey}; pub const W_LABEL: &str = "tinycloud/at-rest/wrap/v1"; const KEK_INFO: &[u8] = b"tinycloud/at-rest/kek/v1"; const NK_INFO: &[u8] = b"tinycloud/at-rest/object-names/v1"; +const DEK_TAG_INFO: &[u8] = b"tinycloud/at-rest/dek-mirror-tag/v1"; const CHAIN_INFO: &[u8] = b"tinycloud/at-rest/ark-chain/v1"; const SEAL_DOMAIN: &[u8] = b"tinycloud/at-rest/ark-seal/v1"; const CHAIN_DOMAIN: &[u8] = b"tinycloud/at-rest/ark-chain-aad/v1"; @@ -179,8 +182,21 @@ impl Kek { /// NK from ARK generation 1; constant for the deployment. pub fn object_name_key(ark_1: &Ark) -> ObjectNameKey { - let nk = hkdf32(ark_1.expose(), NK_INFO); - ObjectNameKey::new(*nk) + ObjectNameKey::from_zeroizing(hkdf32(ark_1.expose(), NK_INFO)) +} + +/// Names a space's mirrored DEK rows without naming the space. From ARK +/// generation 1, so the names survive rotation. +pub struct DekTagKey(Zeroizing<[u8; 32]>); + +impl DekTagKey { + pub fn derive(ark_1: &Ark) -> Self { + Self(hkdf32(ark_1.expose(), DEK_TAG_INFO)) + } + + pub fn tag(&self, space: &str) -> String { + hex::encode(&hkdf32(self.0.as_ref(), space.as_bytes())[..16]) + } } fn dek_aad(space: &str, dek_id: DekId, generation: u32, ark_id: &ArkId) -> Vec { @@ -208,7 +224,7 @@ pub fn new_dek( &dek_aad(space, dek_id, generation, ark_id), bytes.as_ref(), ); - (wrapped, DataKey::new(dek_id, *bytes)) + (wrapped, DataKey::from_zeroizing(dek_id, bytes)) } pub fn unwrap_dek( @@ -220,7 +236,7 @@ pub fn unwrap_dek( ark_id: &ArkId, ) -> Result { let bytes = open32(&kek.0, &dek_aad(space, dek_id, generation, ark_id), wrapped)?; - Ok(DataKey::new(dek_id, *bytes)) + Ok(DataKey::from_zeroizing(dek_id, bytes)) } #[cfg(test)] diff --git a/tinycloud-core/src/at_rest/key.rs b/tinycloud-core/src/at_rest/key.rs index 3a9c61d4..1ff70aa1 100644 --- a/tinycloud-core/src/at_rest/key.rs +++ b/tinycloud-core/src/at_rest/key.rs @@ -33,6 +33,11 @@ impl DataKey { } } + /// Take key bytes already in a zeroizing buffer, with no plain copy. + pub(crate) fn from_zeroizing(id: DekId, bytes: Zeroizing<[u8; 32]>) -> Self { + Self { id, bytes } + } + pub fn id(&self) -> DekId { self.id } @@ -60,6 +65,11 @@ impl ObjectNameKey { Self(Zeroizing::new(bytes)) } + /// Take key bytes already in a zeroizing buffer, with no plain copy. + pub(crate) fn from_zeroizing(bytes: Zeroizing<[u8; 32]>) -> Self { + Self(bytes) + } + pub(crate) fn bytes(&self) -> &[u8; 32] { &self.0 } diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index c27063aa..a6cf4321 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -18,12 +18,12 @@ use tinycloud_at_rest_escrow::{ }; use super::hierarchy::{ - chain_ark, new_dek, object_name_key, open_ark, open_chain, seal_ark, unwrap_dek, Kek, WrapKey, - W_LABEL, + chain_ark, new_dek, object_name_key, open_ark, open_chain, seal_ark, unwrap_dek, DekTagKey, + Kek, WrapKey, W_LABEL, }; use super::store::{ - ark_mirror_name, escrow_mirror_name, AtRestStore, KeyMirror, MirroredArk, ARK_MIRROR_PREFIX, - MIRROR_PREFIX, + ark_mirror_name, dek_mirror_name, escrow_mirror_name, AtRestStore, KeyMirror, MirroredArk, + MirroredDek, ARK_MIRROR_PREFIX, MIRROR_PREFIX, }; use super::{DataKey, DataKeyError, DataKeyProvider, DekId, ObjectNameKey}; use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; @@ -58,6 +58,8 @@ pub enum AtRestError { Mirror(String), #[error("at-rest key state is inconsistent: {0}")] Inconsistent(&'static str), + #[error("at-rest key change lost a race: {0}")] + Conflict(&'static str), #[error("at-rest escrow: {0}")] Escrow(#[from] EscrowError), #[error("at-rest keys are locked")] @@ -85,6 +87,10 @@ pub enum GateRefusal { PinMismatch { pinned: String, efp: String }, #[error("escrow {efp} has no valid verification record; run the ceremony test-unlock")] NotVerified { efp: String }, + /// Not a configuration problem: the store, mirror or stored escrow could + /// not be read. Fix that (or retry); don't re-run the ceremony. + #[error("could not evaluate the escrow: {0}")] + Unavailable(String), } /// How a passing gate was satisfied. @@ -150,11 +156,15 @@ struct Generation { struct Keys { gens: BTreeMap, nk: ObjectNameKey, + dek_tag: DekTagKey, } impl Keys { fn newest(&self) -> (u32, &Generation) { - let (g, gen) = self.gens.last_key_value().expect("at least generation 1"); + let (g, gen) = self + .gens + .last_key_value() + .expect("keys_from_newest loads โ‰ฅ 1 generation"); (*g, gen) } } @@ -228,15 +238,30 @@ fn backend(e: impl std::error::Error + Send + Sync + 'static) -> DataKeyError { DataKeyError::Backend(Box::new(e)) } -/// Open every generation from the rows: the newest under W, older ones via -/// their chain (or W). `Ok(None)` means the newest doesn't open under W. -fn unlock_rows(w: &WrapKey, rows: &[at_rest_key::Model]) -> Result, AtRestError> { +/// Generations must be exactly 1..=n. +fn check_contiguous(rows: &[at_rest_key::Model]) -> Result<(), AtRestError> { + if rows.is_empty() { + return Err(AtRestError::NotProvisioned); + } for (i, row) in rows.iter().enumerate() { if row.generation != i as i32 + 1 { return Err(AtRestError::Inconsistent("ARK generations are not 1..n")); } } - let newest = rows.last().expect("caller checked non-empty"); + Ok(()) +} + +/// A generation whose chain had to be rewritten: (generation, new chain). +type Rechain = (u32, Vec); + +/// Open every generation from the rows under this node's W. `Ok(None)` +/// means the newest doesn't open under W (locked). +fn unlock_rows( + w: &WrapKey, + rows: &[at_rest_key::Model], +) -> Result)>, AtRestError> { + check_contiguous(rows)?; + let newest = rows.last().expect("check_contiguous: non-empty"); let newest_id = ark_id_of(newest)?; let Ok(ark) = open_ark( w, @@ -247,31 +272,53 @@ fn unlock_rows(w: &WrapKey, rows: &[at_rest_key::Model]) -> Result, ) else { return Ok(None); }; - keys_from_newest(w, rows, ark).map(Some) + keys_from_newest(rows, ark, Some(w)).map(Some) } +/// Walk down from the newest ARK. Every older generation must open **by its +/// chain**: that is the only path recovery has, so a broken chain is never +/// accepted silently. +/// +/// With `repair_w` (boot under the root that sealed the rows), a chain that +/// is missing or doesn't open is rebuilt if the generation opens under W, +/// and returned for the caller to store. Without it (recovery under a new +/// root), a broken chain is an error. fn keys_from_newest( - w: &WrapKey, rows: &[at_rest_key::Model], newest_ark: Ark, -) -> Result { + repair_w: Option<&WrapKey>, +) -> Result<(Keys, Vec), AtRestError> { + check_contiguous(rows)?; let mut gens = BTreeMap::new(); + let mut rechain = Vec::new(); let mut newer: Option<(Ark, ArkId)> = None; - let mut ark = Some(newest_ark); for row in rows.iter().rev() { let id = ark_id_of(row)?; let g = row.generation as u32; - let this = match ark.take() { - Some(a) => a, - None => { - let (n_ark, n_id) = newer.as_ref().expect("set after the newest"); - row.chain + let this = match &newer { + None => newest_ark.clone(), + Some((n_ark, n_id)) => { + let by_chain = row + .chain .as_deref() - .and_then(|c| open_chain(n_ark, n_id, c, &id, g).ok()) - .or_else(|| open_ark(w, &row.sealed, &id, g, &row.w_label).ok()) - .ok_or(AtRestError::Inconsistent( - "an older ARK generation opens neither by chain nor by W", - ))? + .and_then(|c| open_chain(n_ark, n_id, c, &id, g).ok()); + match (by_chain, repair_w) { + (Some(ark), _) => ark, + (None, Some(w)) => { + let ark = open_ark(w, &row.sealed, &id, g, &row.w_label).map_err(|_| { + AtRestError::Inconsistent( + "an older ARK generation opens neither by its chain nor under W", + ) + })?; + rechain.push((g, chain_ark(n_ark, n_id, &ark, &id, g))); + ark + } + (None, None) => { + return Err(AtRestError::Inconsistent( + "an older ARK generation's chain is missing or does not open", + )) + } + } } }; newer = Some((this.clone(), id)); @@ -284,8 +331,13 @@ fn keys_from_newest( }, ); } - let nk = object_name_key(&gens.get(&1).expect("generation 1 exists").ark); - Ok(Keys { gens, nk }) + let ark_1 = &gens.get(&1).expect("check_contiguous: generation 1").ark; + let keys = Keys { + nk: object_name_key(ark_1), + dek_tag: DekTagKey::derive(ark_1), + gens, + }; + Ok((keys, rechain)) } pub struct AtRestKeyring { @@ -296,6 +348,9 @@ pub struct AtRestKeyring { phase: RwLock, deks: Mutex>, current: Mutex>, + /// The EFP last confirmed present in the mirror, so `/info` and the gate + /// don't GET it every call. + escrow_mirrored: Mutex>, intake: tokio::sync::Mutex, dek_create: tokio::sync::Mutex<()>, } @@ -303,13 +358,16 @@ pub struct AtRestKeyring { impl AtRestKeyring { /// Load the keyring at boot. /// - /// - Rows open under this root's W: unlocked. + /// - Mirrored ARK generations missing from the database (a restore to + /// an earlier point) are put back first. + /// - Rows open under this root's W: unlocked. A chain that is missing or + /// doesn't open is rebuilt and logged. /// - Rows exist but don't open: **locked** (data intact; recover with two /// shares). - /// - No rows: restore them from the mirror if it has any; otherwise - /// provision generation 1 if `config.provision`, else not provisioned. - /// A store or mirror error fails the call; it never provisions over - /// uncertainty, and never when data keys exist without an ARK. + /// - No rows: provision generation 1 if `config.provision`, else not + /// provisioned. Never when any sealed material exists without an ARK + /// (data keys, escrow rows, any mirror object), and never after a + /// store or mirror error. pub async fn open( store: AtRestStore, mirror: Arc, @@ -317,17 +375,26 @@ impl AtRestKeyring { config: KeyringConfig, ) -> Result { let w = WrapKey::derive(root); + Self::restore_rows(&store, &*mirror).await?; let mut rows = store.keys().await?; if rows.is_empty() { - rows = Self::restore_rows(&store, &*mirror).await?; - } - if rows.is_empty() { - if store.has_space_keys().await? { - return Err(AtRestError::Inconsistent( - "data keys exist but no ARK row or mirrored ARK was found", - )); - } - if config.provision { + if store.has_space_keys().await? + || store.has_escrow_rows().await? + || !mirror + .list(MIRROR_PREFIX) + .await + .map_err(mirror_err)? + .is_empty() + { + // Another process may have provisioned since we read: its + // ARK row is written before anything else. + rows = store.keys().await?; + if rows.is_empty() { + return Err(AtRestError::Inconsistent( + "sealed at-rest material exists but no ARK row or mirrored ARK was found", + )); + } + } else if config.provision { let ark = Ark::generate(); let ark_id = ArkId::generate(); store @@ -342,14 +409,20 @@ impl AtRestKeyring { .await?; // Another process may have won the insert; use what's stored. rows = store.keys().await?; - tracing::info!(generation = 1, %ark_id, "at-rest root key provisioned"); + if rows.first().map(|r| r.ark_id.as_slice()) == Some(&ark_id.0[..]) { + tracing::info!(generation = 1, %ark_id, "at-rest root key provisioned"); + } } } let phase = if rows.is_empty() { Phase::NotProvisioned } else { match unlock_rows(&w, &rows)? { - Some(keys) => { + Some((keys, rechain)) => { + if !rechain.is_empty() { + Self::store_rechain(&store, rechain).await?; + rows = store.keys().await?; + } for row in &rows { Self::mirror_row(&*mirror, row).await?; } @@ -380,6 +453,7 @@ impl AtRestKeyring { current: Mutex::new(LruCache::new( NonZeroUsize::new(DEK_CACHE).expect("non-zero"), )), + escrow_mirrored: Mutex::new(None), intake: tokio::sync::Mutex::new(ShareIntake::new()), dek_create: tokio::sync::Mutex::new(()), }; @@ -389,15 +463,32 @@ impl AtRestKeyring { Ok(keyring) } - async fn restore_rows( - store: &AtRestStore, - mirror: &dyn KeyMirror, - ) -> Result, AtRestError> { + async fn store_rechain(store: &AtRestStore, rechain: Vec) -> Result<(), AtRestError> { + for (g, chain) in rechain { + tracing::warn!( + generation = g, + "at-rest ARK chain was missing or did not open; rebuilt it from the newer generation" + ); + store.set_chain(g as i32, chain).await?; + } + Ok(()) + } + + /// Put back mirrored ARK generations the database lacks, and chains the + /// database lost. Both are AEAD-checked when used, so a bad mirror + /// object can make the node fail closed but never feeds it a key. + async fn restore_rows(store: &AtRestStore, mirror: &dyn KeyMirror) -> Result<(), AtRestError> { let mut names = mirror.list(ARK_MIRROR_PREFIX).await.map_err(mirror_err)?; if names.is_empty() { - return Ok(Vec::new()); + return Ok(()); } names.sort(); + let have: BTreeMap = store + .keys() + .await? + .into_iter() + .map(|r| (r.generation, r)) + .collect(); for name in names { let bytes = mirror @@ -411,10 +502,29 @@ impl AtRestKeyring { .map_err(mirror_err)? .into_row() .map_err(mirror_err)?; - store.insert_key(row).await?; + match have.get(&row.generation) { + None => { + tracing::warn!( + generation = row.generation, + "at-rest ARK row was missing from the database; restored from the mirror" + ); + store.insert_key(row).await?; + } + Some(db) + if db.chain.is_none() && row.chain.is_some() && db.ark_id == row.ark_id => + { + tracing::warn!( + generation = row.generation, + "at-rest ARK chain was missing from the database; restored from the mirror" + ); + store + .set_chain(row.generation, row.chain.expect("checked")) + .await?; + } + Some(_) => {} + } } - tracing::warn!("at-rest ARK rows were missing from the database; restored from the mirror"); - Ok(store.keys().await?) + Ok(()) } async fn mirror_row( @@ -446,9 +556,7 @@ impl AtRestKeyring { } fn parse_bundle(row: &at_rest_escrow::Model) -> Result { - let bundle: EscrowBundle = serde_json::from_str(&row.bundle) - .map_err(|_| AtRestError::Inconsistent("stored escrow bundle is not valid JSON"))?; - bundle.validate()?; + let bundle = EscrowBundle::from_json(row.bundle.as_bytes())?; if bundle.fingerprint().to_string() != row.efp { return Err(AtRestError::Inconsistent( "stored escrow bundle does not match its fingerprint", @@ -499,6 +607,9 @@ impl AtRestKeyring { } async fn mirror_escrow(&self, row: &at_rest_escrow::Model) -> Result<(), AtRestError> { + if self.escrow_mirrored.lock().as_deref() == Some(row.efp.as_str()) { + return Ok(()); + } let name = escrow_mirror_name(&row.efp); if self.mirror.get(&name).await.map_err(mirror_err)?.is_none() { self.mirror @@ -506,6 +617,7 @@ impl AtRestKeyring { .await .map_err(mirror_err)?; } + *self.escrow_mirrored.lock() = Some(row.efp.clone()); Ok(()) } @@ -527,9 +639,7 @@ impl AtRestKeyring { else { return Ok(None); }; - let bundle: EscrowBundle = serde_json::from_slice(&bytes) - .map_err(|_| AtRestError::Inconsistent("mirrored escrow bundle is not valid JSON"))?; - bundle.validate()?; + let bundle = EscrowBundle::from_json(&bytes)?; Ok((bundle.fingerprint() == pinned).then_some(bundle)) } @@ -618,9 +728,9 @@ impl AtRestKeyring { let Some(json) = row.record else { return Ok(None); }; - let Ok(record) = serde_json::from_str::(&json) else { - return Ok(None); - }; + // A record this node can't read (newer version, corrupt) is an + // error to surface, not "unverified". + let record = VerificationRecord::from_json(json.as_bytes())?; Ok(record.verify(&gen.ark, bundle).is_ok().then_some(record)) } @@ -642,7 +752,8 @@ impl AtRestKeyring { } let bundle = match self.ensure_escrow().await { Ok(Some(b)) => b, - _ => return Err(GateRefusal::NoEscrow), + Ok(None) => return Err(GateRefusal::NoEscrow), + Err(e) => return Err(GateRefusal::Unavailable(e.to_string())), }; if bundle.holders() != self.config.holders { return Err(GateRefusal::RecipientsMismatch); @@ -670,9 +781,10 @@ impl AtRestKeyring { } match self.verified_record(&bundle).await { Ok(Some(_)) => Ok(GatePass::Verified { efp }), - _ => Err(GateRefusal::NotVerified { + Ok(None) => Err(GateRefusal::NotVerified { efp: efp.to_string(), }), + Err(e) => Err(GateRefusal::Unavailable(e.to_string())), } } @@ -761,7 +873,9 @@ impl AtRestKeyring { "recovered ARK is not the newest generation", )); } - let keys = keys_from_newest(&self.w, &rows, ark)?; + // Under a new root the rows still carry the old W: every older + // generation must open by its chain, or recovery fails loudly. + let (keys, _) = keys_from_newest(&rows, ark, None)?; for (g, gen) in &keys.gens { self.store .reseal_key( @@ -785,38 +899,155 @@ impl AtRestKeyring { /// Start ARK generation n+1. Older generations stay readable through the /// chain; new DEKs wrap under the new KEK. The new generation has no /// verified escrow until a ceremony runs, so the gate closes. + /// + /// The new row and the older generation's chain are written in one + /// transaction. If another node rotated first, nothing is written, this + /// keyring reloads what is stored, and the call returns + /// [`AtRestError::Conflict`]. pub async fn rotate_ark(&self) -> Result { let keys = self.keys()?; let (g, old) = keys.newest(); let next = g + 1; let ark = Ark::generate(); let ark_id = ArkId::generate(); - self.store - .insert_key(at_rest_key::Model { - generation: next as i32, - ark_id: ark_id.0.to_vec(), - w_label: W_LABEL.into(), - sealed: seal_ark(&self.w, &ark, &ark_id, next), - chain: None, - created_at: now_rfc3339(), - }) - .await?; - self.store - .set_chain(g as i32, chain_ark(&ark, &ark_id, &old.ark, &old.ark_id, g)) + let won = self + .store + .rotate( + at_rest_key::Model { + generation: next as i32, + ark_id: ark_id.0.to_vec(), + w_label: W_LABEL.into(), + sealed: seal_ark(&self.w, &ark, &ark_id, next), + chain: None, + created_at: now_rfc3339(), + }, + chain_ark(&ark, &ark_id, &old.ark, &old.ark_id, g), + ) .await?; let rows = self.store.keys().await?; - let keys = unlock_rows(&self.w, &rows)?.ok_or(AtRestError::Inconsistent( - "a just-written generation does not open", + let (keys, rechain) = unlock_rows(&self.w, &rows)?.ok_or(AtRestError::Inconsistent( + "the stored newest generation does not open under this node's root", ))?; - for row in &rows { + Self::store_rechain(&self.store, rechain).await?; + for row in &self.store.keys().await? { Self::mirror_row(&*self.mirror, row).await?; } *self.phase.write() = Phase::Unlocked(Arc::new(keys)); self.current.lock().clear(); self.ensure_escrow().await?; + if !won { + return Err(AtRestError::Conflict( + "another node rotated the ARK first; reloaded the stored generations", + )); + } + tracing::warn!(generation = next, %ark_id, "at-rest ARK rotated"); Ok(next) } + /// A space's DEK row from the mirror, put back into the database. + async fn restore_dek( + &self, + keys: &Keys, + space: &str, + dek_id: u32, + ) -> Result, DataKeyError> { + let name = dek_mirror_name(&keys.dek_tag.tag(space), dek_id); + let Some(bytes) = self + .mirror + .get(&name) + .await + .map_err(|e| backend(mirror_err(e)))? + else { + return Ok(None); + }; + let row = serde_json::from_slice::(&bytes) + .map_err(|e| backend(mirror_err(e)))? + .into_row(space) + .map_err(|e| backend(mirror_err(e)))?; + if row.dek_id as u32 != dek_id { + return Err(backend(AtRestError::Inconsistent( + "mirrored data key has a different id than its name", + ))); + } + tracing::warn!( + dek_id, + "at-rest data key was missing from the database; restored from the mirror" + ); + self.store.insert_dek(row).await.map_err(backend)?; + let id = i32::try_from(dek_id).map_err(|_| DataKeyError::UnknownKey { + dek_id: DekId::new(dek_id), + })?; + self.store.dek(space, id).await.map_err(backend) + } + + /// Make sure the mirror holds exactly this DEK row before any object is + /// sealed under it. A different mirrored row with the same id means + /// objects may exist under another key: refuse, never overwrite. + async fn mirror_dek( + &self, + keys: &Keys, + row: &space_data_key::Model, + ) -> Result<(), DataKeyError> { + let name = dek_mirror_name(&keys.dek_tag.tag(&row.space), row.dek_id as u32); + match self + .mirror + .get(&name) + .await + .map_err(|e| backend(mirror_err(e)))? + { + None => { + let body = serde_json::to_vec(&MirroredDek::from_row(row)).expect("serializable"); + self.mirror + .put(&name, body) + .await + .map_err(|e| backend(mirror_err(e))) + } + Some(bytes) => { + let same = serde_json::from_slice::(&bytes) + .ok() + .and_then(|m| m.into_row(&row.space).ok()) + .is_some_and(|m| { + m.dek_id == row.dek_id + && m.generation == row.generation + && m.wrapped == row.wrapped + }); + if same { + Ok(()) + } else { + Err(backend(AtRestError::Inconsistent( + "the mirror holds a different data key with this id", + ))) + } + } + } + } + + /// Generate, store and return the space's first DEK row. If another + /// process won the insert, its row is returned. + async fn create_dek( + &self, + keys: &Keys, + space: &str, + ) -> Result { + let (g, gen) = keys.newest(); + let (wrapped, _) = new_dek(&gen.kek, space, DekId::new(1), g, &gen.ark_id); + self.store + .insert_dek(space_data_key::Model { + space: space.to_owned(), + dek_id: 1, + generation: g as i32, + wrapped, + created_at: now_rfc3339(), + }) + .await + .map_err(backend)?; + self.store + .dek(space, 1) + .await + .map_err(backend)? + .ok_or_else(|| backend(AtRestError::Inconsistent("inserted data key is missing"))) + } + async fn load_dek( &self, keys: &Keys, @@ -865,32 +1096,17 @@ impl DataKeyProvider for AtRestKeyring { let _create = self.dek_create.lock().await; match self.store.newest_dek(space).await.map_err(backend)? { Some(row) => row, - None => { - let (g, gen) = keys.newest(); - let id = DekId::new(1); - let (wrapped, _) = new_dek(&gen.kek, space, id, g, &gen.ark_id); - self.store - .insert_dek(space_data_key::Model { - space: space.to_owned(), - dek_id: 1, - generation: g as i32, - wrapped, - created_at: now_rfc3339(), - }) - .await - .map_err(backend)?; - // Another process may have won; serve whatever is stored. - self.store - .dek(space, 1) - .await - .map_err(backend)? - .ok_or_else(|| { - backend(AtRestError::Inconsistent("inserted data key is missing")) - })? - } + // A database restored to before this space's first DEK: + // objects may still be sealed under the mirrored one. + None => match self.restore_dek(&keys, space, 1).await? { + Some(row) => row, + None => self.create_dek(&keys, space).await?, + }, } } }; + // First use of this space's current key in this process. + self.mirror_dek(&keys, &row).await?; let key = self.load_dek(&keys, &row).await?; self.current.lock().put(space.to_owned(), key.id()); Ok(key) @@ -904,7 +1120,11 @@ impl DataKeyProvider for AtRestKeyring { let Ok(id) = i32::try_from(dek_id.get()) else { return Err(DataKeyError::UnknownKey { dek_id }); }; - match self.store.dek(space, id).await.map_err(backend)? { + let row = match self.store.dek(space, id).await.map_err(backend)? { + Some(row) => Some(row), + None => self.restore_dek(&keys, space, dek_id.get()).await?, + }; + match row { Some(row) => self.load_dek(&keys, &row).await, None => Err(DataKeyError::UnknownKey { dek_id }), } diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index 71bdcf8c..81b6321b 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -1,7 +1,9 @@ use std::io::Read; use std::sync::Arc; -use sea_orm::{ConnectOptions, Database, DatabaseConnection, EntityTrait}; +use sea_orm::{ + ColumnTrait, ConnectOptions, Database, DatabaseConnection, EntityTrait, QueryFilter, +}; use sea_orm_migration::MigratorTrait; use tinycloud_at_rest_escrow::{wrap_share, EscrowBundle, HolderRecipient, SharePayload}; @@ -412,6 +414,18 @@ async fn gate_requires_pinned_verified_escrow_for_the_configured_recipients() { k.gate(false).await, Err(GateRefusal::NotVerified { .. }) )); + // A record this node can't read (a newer format) is not "unverified": + // the operator must fix the node, not re-run the ceremony. + let mut newer: serde_json::Value = serde_json::from_str(&record).unwrap(); + newer["version"] = serde_json::json!(2); + AtRestStore::new(db.clone()) + .save_record(&efp.to_string(), newer.to_string()) + .await + .unwrap(); + assert!(matches!( + k.gate(false).await, + Err(GateRefusal::Unavailable(e)) if e.contains("version 2") + )); AtRestStore::new(db.clone()) .save_record(&efp.to_string(), record) .await @@ -546,10 +560,10 @@ async fn share_intake_is_rate_limited_per_boot() { )); } -#[tokio::test] -async fn postgres_two_nodes_converge_on_one_ark_and_one_dek_per_space() { +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn postgres_concurrent_writers_converge_on_one_row_each() { let Some(pg) = crate::test_support::PostgresTestSchema::new( - "postgres_two_nodes_converge_on_one_ark_and_one_dek_per_space", + "postgres_concurrent_writers_converge_on_one_row_each", ) .await else { @@ -598,4 +612,333 @@ async fn postgres_two_nodes_converge_on_one_ark_and_one_dek_per_space() { .len(), 4 ); + + // Store conflicts keep the first row, deterministically, on PG too. + store_conflicts_keep_the_first_row(&pg.conn).await; + + // Concurrent re-splits of one generation leave exactly one active bundle. + let store = AtRestStore::new(pg.conn.clone()); + for round in 0..50 { + let row = |n: u8| at_rest_escrow::Model { + efp: format!("{round:04}-{n}"), + generation: 1, + bundle: "{}".into(), + active: true, + record: None, + created_at: now_rfc3339(), + }; + let (x, y) = tokio::join!(store.activate_escrow(row(1)), store.activate_escrow(row(2))); + x.unwrap(); + y.unwrap(); + let active = at_rest_escrow::Entity::find() + .all(&pg.conn) + .await + .unwrap() + .into_iter() + .filter(|b| b.generation == 1 && b.active) + .count(); + assert_eq!(active, 1, "round {round}"); + } +} + +/// Both conflict branches return `Ok` and keep the first row; a rotation +/// onto an existing generation writes nothing. +async fn store_conflicts_keep_the_first_row(db: &DatabaseConnection) { + let store = AtRestStore::new(db.clone()); + let key = |ark_id: u8, generation: i32| at_rest_key::Model { + generation, + ark_id: vec![ark_id; 16], + w_label: W_LABEL.into(), + sealed: vec![ark_id; 60], + chain: None, + created_at: now_rfc3339(), + }; + let gen1 = store.keys().await.unwrap()[0].clone(); + store.insert_key(key(0xee, 1)).await.unwrap(); + assert_eq!(store.keys().await.unwrap()[0], gen1); + + let dek = |wrapped: u8| space_data_key::Model { + space: "conflict".into(), + dek_id: 1, + generation: 1, + wrapped: vec![wrapped; 60], + created_at: now_rfc3339(), + }; + store.insert_dek(dek(1)).await.unwrap(); + store.insert_dek(dek(2)).await.unwrap(); + assert_eq!( + store.dek("conflict", 1).await.unwrap().unwrap().wrapped, + vec![1; 60] + ); + + // Generation 1 exists: rotating "to" it writes neither row nor chain. + let before = store.keys().await.unwrap(); + assert!(!store.rotate(key(0xdd, 1), vec![1; 60]).await.unwrap()); + assert_eq!(store.keys().await.unwrap(), before); +} + +#[tokio::test] +async fn sqlite_store_conflicts_keep_the_first_row() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + store_conflicts_keep_the_first_row(&db).await; +} + +async fn set_chain(db: &DatabaseConnection, generation: i32, chain: Option>) { + at_rest_key::Entity::update_many() + .col_expr(at_rest_key::Column::Chain, chain.into()) + .filter(at_rest_key::Column::Generation.eq(generation)) + .exec(db) + .await + .unwrap(); +} + +fn chain_of(rows: &[at_rest_key::Model], generation: i32) -> Option> { + rows.iter() + .find(|r| r.generation == generation) + .unwrap() + .chain + .clone() +} + +#[tokio::test] +async fn a_missing_or_foreign_chain_is_rebuilt_under_the_same_root() { + for broken in [None, Some(vec![0x5a; 60])] { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let old = k.current_key("space-a").await.unwrap(); + k.rotate_ark().await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + // A crash or a stale writer left generation 1 without a usable + // chain, and the mirror has no better copy. + set_chain(&db, 1, broken.clone()).await; + mirror.remove_prefix(ARK_MIRROR_PREFIX); + + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + let rows = AtRestStore::new(db.clone()).keys().await.unwrap(); + let rebuilt = chain_of(&rows, 1).expect("chain rebuilt"); + assert_ne!(Some(rebuilt), broken); + drop(k); + + // The rebuilt chain is what recovery under a new root relies on. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await + .unwrap(); + submit(&k, &h, 0).await.unwrap(); + assert_eq!(submit(&k, &h, 2).await.unwrap(), ShareOutcome::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + old.bytes() + ); + } +} + +#[tokio::test] +async fn recovery_fails_loudly_on_a_broken_chain_or_missing_generation() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + k.current_key("space-a").await.unwrap(); + k.rotate_ark().await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + set_chain(&db, 1, None).await; + mirror.remove_prefix(ARK_MIRROR_PREFIX); + + // Under a new root nothing can rebuild the chain: two valid shares are + // refused with the reason, and the node stays locked. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await + .unwrap(); + submit(&k, &h, 0).await.unwrap(); + assert!(matches!( + submit(&k, &h, 1).await, + Err(ShareRefusal::Keyring(AtRestError::Inconsistent(_))) + )); + assert_eq!(k.state(), KeyringState::Locked); + + // Generation 1 deleted after boot: an error, not a panic. The two + // valid shares stay held, so any submission retries the unlock. + at_rest_key::Entity::delete_by_id(1) + .exec(&db) + .await + .unwrap(); + assert!(matches!( + submit(&k, &h, 0).await, + Err(ShareRefusal::Keyring(AtRestError::Inconsistent(_))) + )); + assert_eq!(k.state(), KeyringState::Locked); +} + +#[tokio::test] +async fn a_stale_node_cannot_rotate_over_another_nodes_rotation() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let a = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let b = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let old = a.current_key("space-a").await.unwrap(); + assert_eq!(a.rotate_ark().await.unwrap(), 2); + let chain = chain_of(&AtRestStore::new(db.clone()).keys().await.unwrap(), 1); + + // B still believes generation 1 is newest. + assert!(matches!( + b.rotate_ark().await, + Err(AtRestError::Conflict(_)) + )); + let rows = AtRestStore::new(db.clone()).keys().await.unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(chain_of(&rows, 1), chain, "A's chain is untouched"); + // B reloaded A's generation: new spaces wrap under generation 2. + b.current_key("space-b").await.unwrap(); + assert_eq!( + AtRestStore::new(db.clone()) + .dek("space-b", 1) + .await + .unwrap() + .unwrap() + .generation, + 2 + ); + assert_eq!(b.rotate_ark().await.unwrap(), 3); + let efp = b.escrow_view().await.unwrap().unwrap().efp; + drop((a, b)); + + // Generation 3's escrow alone recovers every generation. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await + .unwrap(); + submit(&k, &h, 1).await.unwrap(); + assert_eq!(submit(&k, &h, 2).await.unwrap(), ShareOutcome::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + old.bytes() + ); +} + +#[tokio::test] +async fn data_keys_survive_a_restore_to_before_they_were_created() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let a = k.current_key("space-a").await.unwrap(); + // Mirror names don't reveal the space. + assert!(mirror.names().iter().all(|n| !n.contains("space-a"))); + drop(k); + space_data_key::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + + // The restored database lacks the row; objects sealed under it remain. + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + a.bytes() + ); + space_data_key::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.current_key("space-a").await.unwrap().bytes(), a.bytes()); + drop(k); + + // A different DEK under the same id would strand every object sealed + // under the mirrored one: refused, and the mirror is not overwritten. + space_data_key::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + let keys = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let gen = &keys.keys().unwrap(); + let (g, newest) = gen.newest(); + let (wrapped, _) = new_dek(&newest.kek, "space-a", DekId::new(1), g, &newest.ark_id); + AtRestStore::new(db.clone()) + .insert_dek(space_data_key::Model { + space: "space-a".into(), + dek_id: 1, + generation: g as i32, + wrapped, + created_at: now_rfc3339(), + }) + .await + .unwrap(); + assert!(matches!( + keys.current_key("space-a").await, + Err(DataKeyError::Backend(_)) + )); + space_data_key::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.current_key("space-a").await.unwrap().bytes(), a.bytes()); +} + +#[tokio::test] +async fn escrow_rows_or_any_mirror_object_block_provisioning() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + mirror.remove_prefix(ARK_MIRROR_PREFIX); + // Only escrow rows and escrow mirror objects are left. + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::Inconsistent(_)) + )); + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + // Only a mirror object is left. + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::Inconsistent(_)) + )); + assert!(at_rest_key::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn a_newer_mirror_format_is_refused_not_guessed() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + let name = mirror + .names() + .into_iter() + .find(|n| n.starts_with(ARK_MIRROR_PREFIX)) + .unwrap(); + let mut obj: serde_json::Value = + serde_json::from_slice(&mirror.get(&name).await.unwrap().unwrap()).unwrap(); + obj["version"] = serde_json::json!(2); + mirror + .put(&name, serde_json::to_vec(&obj).unwrap()) + .await + .unwrap(); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::Mirror(e)) if e.contains("version 2") + )); +} + +#[tokio::test] +async fn migration_down_refuses_to_drop_keys() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert!(crate::migrations::Migrator::down(&db, Some(1)) + .await + .is_err()); + assert_eq!(at_rest_key::Entity::find().all(&db).await.unwrap().len(), 1); + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + crate::migrations::Migrator::down(&db, Some(1)) + .await + .unwrap(); + assert!(at_rest_key::Entity::find().all(&db).await.is_err()); } diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs index dfd2b8a4..0d23466d 100644 --- a/tinycloud-core/src/at_rest/store.rs +++ b/tinycloud-core/src/at_rest/store.rs @@ -1,15 +1,17 @@ //! Persistence for the at-rest keys: PG/SQLite rows plus a mirror of the -//! sealed ARK rows and escrow bundles in the block store (TC-887 plan ยง5). +//! sealed ARK rows, escrow bundles and wrapped DEKs in the block store +//! (TC-887 plan ยง5). //! -//! The mirror covers a database restored to a point before an ARK row was -//! written: sealed blobs in object storage outlive such a restore. +//! The mirror covers a database restored to a point before a key row was +//! written: sealed blobs in object storage outlive such a restore, and an +//! object sealed under a DEK must never meet a different DEK with its id. use std::sync::Arc; use base64::Engine; use sea_orm::{ sea_query::OnConflict, ActiveValue::Set, ColumnTrait, DbErr, EntityTrait, PaginatorTrait, - QueryFilter, QueryOrder, TransactionTrait, + QueryFilter, QueryOrder, QuerySelect, TransactionTrait, }; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; @@ -40,10 +42,31 @@ pub(crate) fn escrow_mirror_name(efp: &str) -> String { format!("{MIRROR_PREFIX}escrow-{efp}.json") } +/// `tag` is a keyed hash of the space (see `hierarchy::dek_mirror_tag`), so +/// the bucket listing doesn't name spaces. +pub(crate) fn dek_mirror_name(tag: &str, dek_id: u32) -> String { + format!("{MIRROR_PREFIX}dek-{tag}-{dek_id:010}.json") +} + +/// Mirror object format version; a newer one is refused, never guessed at. +const MIRROR_VERSION: u32 = 1; + +fn check_version(version: u32) -> Result<(), MirrorError> { + if version == MIRROR_VERSION { + Ok(()) + } else { + Err(format!( + "mirror object version {version} is not supported (this node reads {MIRROR_VERSION})" + ) + .into()) + } +} + /// The mirrored form of an `at_rest_key` row. #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct MirroredArk { + pub version: u32, pub generation: u32, pub ark_id: String, pub w_label: String, @@ -59,6 +82,7 @@ fn b64() -> base64::engine::GeneralPurpose { impl MirroredArk { pub fn from_row(row: &at_rest_key::Model) -> Self { Self { + version: MIRROR_VERSION, generation: row.generation as u32, ark_id: hex::encode(&row.ark_id), w_label: row.w_label.clone(), @@ -69,6 +93,7 @@ impl MirroredArk { } pub fn into_row(self) -> Result { + check_version(self.version)?; Ok(at_rest_key::Model { generation: i32::try_from(self.generation)?, ark_id: hex::decode(&self.ark_id)?, @@ -80,6 +105,41 @@ impl MirroredArk { } } +/// The mirrored form of a `space_data_key` row. The space is not stored: +/// whoever restores the row knows it, and the wrap's AAD binds it. +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct MirroredDek { + pub version: u32, + pub dek_id: u32, + pub generation: u32, + pub wrapped: String, + pub created_at: String, +} + +impl MirroredDek { + pub fn from_row(row: &space_data_key::Model) -> Self { + Self { + version: MIRROR_VERSION, + dek_id: row.dek_id as u32, + generation: row.generation as u32, + wrapped: b64().encode(&row.wrapped), + created_at: row.created_at.clone(), + } + } + + pub fn into_row(self, space: &str) -> Result { + check_version(self.version)?; + Ok(space_data_key::Model { + space: space.to_owned(), + dek_id: i32::try_from(self.dek_id)?, + generation: i32::try_from(self.generation)?, + wrapped: b64().decode(&self.wrapped)?, + created_at: self.created_at, + }) + } +} + /// Row access. On SQLite, writes take the node's shared writer lock. #[derive(Clone)] pub struct AtRestStore { @@ -160,6 +220,49 @@ impl AtRestStore { .map(|_| ()) } + /// Write generation `row.generation` and the chain that lets it open + /// generation `row.generation - 1`, in one transaction. Returns `false`, + /// writing nothing, if that generation already exists or the older one + /// is already chained: another node rotated first. + pub async fn rotate(&self, row: at_rest_key::Model, chain: Vec) -> Result { + let older = row.generation - 1; + let _w = self.writer().await; + let tx = self.db.begin().await?; + let inserted = at_rest_key::Entity::insert(at_rest_key::ActiveModel { + generation: Set(row.generation), + ark_id: Set(row.ark_id), + w_label: Set(row.w_label), + sealed: Set(row.sealed), + chain: Set(row.chain), + created_at: Set(row.created_at), + }) + .on_conflict( + OnConflict::column(at_rest_key::Column::Generation) + .do_nothing() + .to_owned(), + ) + .exec(&tx) + .await; + match inserted { + Ok(_) => {} + Err(DbErr::RecordNotInserted) => return Ok(false), + Err(e) => return Err(e), + } + let chained = at_rest_key::Entity::update_many() + .col_expr(at_rest_key::Column::Chain, chain.into()) + .filter(at_rest_key::Column::Generation.eq(older)) + .filter(at_rest_key::Column::Chain.is_null()) + .exec(&tx) + .await?; + if chained.rows_affected != 1 { + return Ok(false); + } + tx.commit().await?; + Ok(true) + } + + /// Replace a generation's chain. Only for repairing a chain that is + /// missing or doesn't open, with the newer ARK in hand. pub async fn set_chain(&self, generation: i32, chain: Vec) -> Result<(), DbErr> { let _w = self.writer().await; at_rest_key::Entity::update_many() @@ -170,10 +273,18 @@ impl AtRestStore { .map(|_| ()) } + /// Any escrow row at all (sealed material that must block provisioning). + pub async fn has_escrow_rows(&self) -> Result { + Ok(at_rest_escrow::Entity::find().count(&self.db).await? > 0) + } + pub async fn has_space_keys(&self) -> Result { Ok(space_data_key::Entity::find().count(&self.db).await? > 0) } + /// The generation's active bundle. The partial unique index and + /// `activate_escrow`'s lock keep it unique; the order makes the choice + /// deterministic even if a database lacks the index. pub async fn active_escrow( &self, generation: i32, @@ -181,6 +292,8 @@ impl AtRestStore { at_rest_escrow::Entity::find() .filter(at_rest_escrow::Column::Generation.eq(generation)) .filter(at_rest_escrow::Column::Active.eq(true)) + .order_by_desc(at_rest_escrow::Column::CreatedAt) + .order_by_asc(at_rest_escrow::Column::Efp) .one(&self.db) .await } @@ -192,9 +305,19 @@ impl AtRestStore { } /// Make `row` the generation's only active bundle. Older bundles stay. + /// + /// Concurrent activations serialize on the generation's `at_rest_key` + /// row (`FOR UPDATE` on PG/MySQL; SQLite transactions are serializable + /// and in-process writers share the writer lock), so the deactivate + /// step always sees the other's committed bundle. pub async fn activate_escrow(&self, row: at_rest_escrow::Model) -> Result<(), DbErr> { let _w = self.writer().await; let tx = self.db.begin().await?; + at_rest_key::Entity::find_by_id(row.generation) + .lock_exclusive() + .one(&tx) + .await? + .ok_or_else(|| DbErr::RecordNotFound("escrowed ARK generation".into()))?; at_rest_escrow::Entity::update_many() .col_expr(at_rest_escrow::Column::Active, false.into()) .filter(at_rest_escrow::Column::Generation.eq(row.generation)) @@ -299,6 +422,11 @@ impl MemoryMirror { self.objects.lock().clear(); } + /// Drop every object under `prefix`, like a lost or rolled-back bucket. + pub fn remove_prefix(&self, prefix: &str) { + self.objects.lock().retain(|k, _| !k.starts_with(prefix)); + } + fn check(&self) -> Result<(), MirrorError> { if self.fail.load(std::sync::atomic::Ordering::SeqCst) { Err("mirror unavailable".into()) diff --git a/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs b/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs index 219afcbb..8dbab5f1 100644 --- a/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs +++ b/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs @@ -2,6 +2,7 @@ //! until a node is configured to provision at-rest keys. use sea_orm_migration::prelude::*; +use sea_orm_migration::sea_orm::{DatabaseBackend, EntityTrait, PaginatorTrait}; use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; @@ -77,6 +78,24 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + // At most one active bundle per generation. MySQL has no partial + // indexes (sea-query would emit a plain unique index, which would + // forbid keeping old bundles); there, the row lock taken by + // `AtRestStore::activate_escrow` alone serializes activations. + if manager.get_database_backend() != DatabaseBackend::MySql { + manager + .create_index( + Index::create() + .name("at_rest_escrow_one_active") + .table(at_rest_escrow::Entity) + .col(at_rest_escrow::Column::Generation) + .unique() + .if_not_exists() + .and_where(Expr::col(at_rest_escrow::Column::Active).eq(true)) + .to_owned(), + ) + .await?; + } manager .create_table( Table::create() @@ -117,7 +136,19 @@ impl MigrationTrait for Migration { .await } + /// Refuses while any key row exists: dropping them would leave sealed + /// data depending on the mirror alone, or on nothing. async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + let rows = at_rest_key::Entity::find().count(db).await? + + at_rest_escrow::Entity::find().count(db).await? + + space_data_key::Entity::find().count(db).await?; + if rows > 0 { + return Err(DbErr::Migration( + "refusing to drop the at-rest key tables: they hold keys that exist nowhere else" + .into(), + )); + } manager .drop_table(Table::drop().table(space_data_key::Entity).to_owned()) .await?; From 5bd76560438099fbaf132fb8da68918579c368f9 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 09:11:04 +0000 Subject: [PATCH 04/15] test(TC-887): roll the cursor-index lock test back to its own migration by name The test rolled back one migration assuming it was the newest; with m20261017_000000_at_rest_keys after it, locate it by name and roll back it and every later migration. --- .../m20261015_000000_delegation_cursor_index.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs b/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs index 9083d249..7e03dea5 100644 --- a/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs +++ b/tinycloud-core/src/migrations/m20261015_000000_delegation_cursor_index.rs @@ -106,8 +106,14 @@ mod tests { return; }; let db = &fixture.conn; - // Back to the state a node upgrading to this migration boots with. - crate::migrations::Migrator::down(db, Some(1)) + // Back to the state a node upgrading to this migration boots with: + // roll back this migration and every later one, located by name. + let migrations = crate::migrations::Migrator::migrations(); + let index = migrations + .iter() + .position(|m| m.name() == Migration.name()) + .unwrap(); + crate::migrations::Migrator::down(db, Some((migrations.len() - index) as u32)) .await .unwrap(); assert!(!recorded(db).await); From 8f5e8ab9f8de36d87092a8c616f4b514af5a179d Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 11:05:46 +0000 Subject: [PATCH 05/15] fix(TC-887): store only authenticated mirror rows; pin before re-split; free junk shares (#326) - Boot collects ARK candidates per generation (database row first, then mirror rows) and unlocks from the newest that opens under W with a chain to generation 1. Only rows on that path are stored, resealed under W; forged, junk, gapped or oversized mirror objects are skipped with a warning. A locked node waits for the pinned bundle's generation and stores mirror rows only once the recovered ARK opens them. - restore_dek stores a mirrored DEK only if its wrap authenticates for the space and id; mirror_dek refuses only a genuine different DEK and replaces objects that don't authenticate. - Mirror reads are capped (MAX_MIRROR_OBJECT, MirrorObjectTooLarge); an oversized object counts as absent. - With verified_fp pinned the keyring never re-splits: it uses the pinned bundle (restored from the mirror if the database lost it) or fails with EscrowPinned. - A share spends the per-boot budget only after it decrypts and matches a commitment. - Tests: forged generation, gap and junk on a healthy database; empty database under the same and a new root; a copied DEK object; pinned re-split; junk never spends the budget. --- test/m1-realdata-e2e/Cargo.lock | 36 + test/w5-policy-runtime-node-e2e/Cargo.lock | 36 + tinycloud-core/src/at_rest/keyring.rs | 743 ++++++++++++-------- tinycloud-core/src/at_rest/keyring/tests.rs | 248 ++++++- tinycloud-core/src/at_rest/store.rs | 15 +- 5 files changed, 779 insertions(+), 299 deletions(-) diff --git a/test/m1-realdata-e2e/Cargo.lock b/test/m1-realdata-e2e/Cargo.lock index e42fc069..0df2147a 100644 --- a/test/m1-realdata-e2e/Cargo.lock +++ b/test/m1-realdata-e2e/Cargo.lock @@ -233,6 +233,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayref" version = "0.3.9" @@ -2823,6 +2829,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -2841,6 +2856,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heapless" version = "0.8.0" @@ -8343,6 +8367,7 @@ dependencies = [ "subtle", "thiserror 2.0.18", "vsss-rs", + "yaml-rust2", "zeroize", ] @@ -9778,6 +9803,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "yaml-rust2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e5b818a27a4cd30884ea380857a5e56f7ec3ba24a3990a3cc0b95af3238e18" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink 0.12.2", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/test/w5-policy-runtime-node-e2e/Cargo.lock b/test/w5-policy-runtime-node-e2e/Cargo.lock index f0309708..af07dd82 100644 --- a/test/w5-policy-runtime-node-e2e/Cargo.lock +++ b/test/w5-policy-runtime-node-e2e/Cargo.lock @@ -244,6 +244,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayref" version = "0.3.9" @@ -3114,6 +3120,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -3132,6 +3147,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heapless" version = "0.8.0" @@ -8747,6 +8771,7 @@ dependencies = [ "subtle", "thiserror 2.0.18", "vsss-rs", + "yaml-rust2", "zeroize", ] @@ -10214,6 +10239,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "yaml-rust2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e5b818a27a4cd30884ea380857a5e56f7ec3ba24a3990a3cc0b95af3238e18" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink 0.12.2", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index a6cf4321..77cbb1f3 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -22,8 +22,8 @@ use super::hierarchy::{ Kek, WrapKey, W_LABEL, }; use super::store::{ - ark_mirror_name, dek_mirror_name, escrow_mirror_name, AtRestStore, KeyMirror, MirroredArk, - MirroredDek, ARK_MIRROR_PREFIX, MIRROR_PREFIX, + ark_mirror_name, dek_mirror_name, escrow_mirror_name, AtRestStore, KeyMirror, + MirrorObjectTooLarge, MirroredArk, MirroredDek, ARK_MIRROR_PREFIX, MIRROR_PREFIX, }; use super::{DataKey, DataKeyError, DataKeyProvider, DekId, ObjectNameKey}; use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; @@ -60,6 +60,8 @@ pub enum AtRestError { Inconsistent(&'static str), #[error("at-rest key change lost a race: {0}")] Conflict(&'static str), + #[error("at_rest.escrow.verified_fp pins an escrow that is not for the configured recipients and the newest key; the node never re-splits while a fingerprint is pinned (remove the pin in the attested compose to change holders)")] + EscrowPinned, #[error("at-rest escrow: {0}")] Escrow(#[from] EscrowError), #[error("at-rest keys are locked")] @@ -226,6 +228,21 @@ fn mirror_err(e: impl std::fmt::Display) -> AtRestError { AtRestError::Mirror(e.to_string()) } +/// A mirror read where an oversized object counts as absent (it can't be a +/// key object) rather than an error that would stop the caller. +async fn get_capped( + mirror: &dyn KeyMirror, + name: &str, +) -> Result>, super::store::MirrorError> { + match mirror.get(name).await { + Err(e) if e.is::() => { + tracing::warn!(object = ?name, "ignoring an oversized at-rest mirror object"); + Ok(None) + } + other => other, + } +} + fn ark_id_of(row: &at_rest_key::Model) -> Result { row.ark_id .as_slice() @@ -240,9 +257,6 @@ fn backend(e: impl std::error::Error + Send + Sync + 'static) -> DataKeyError { /// Generations must be exactly 1..=n. fn check_contiguous(rows: &[at_rest_key::Model]) -> Result<(), AtRestError> { - if rows.is_empty() { - return Err(AtRestError::NotProvisioned); - } for (i, row) in rows.iter().enumerate() { if row.generation != i as i32 + 1 { return Err(AtRestError::Inconsistent("ARK generations are not 1..n")); @@ -251,93 +265,259 @@ fn check_contiguous(rows: &[at_rest_key::Model]) -> Result<(), AtRestError> { Ok(()) } -/// A generation whose chain had to be rewritten: (generation, new chain). -type Rechain = (u32, Vec); +/// A row that may hold generation `row.generation`'s ARK. Mirror rows are +/// unauthenticated until a chain from an authenticated ARK opens them. +#[derive(Clone)] +struct Candidate { + row: at_rest_key::Model, + from_db: bool, +} + +/// Candidates per generation: the database row first, then mirrored rows +/// that differ from it. +type Candidates = BTreeMap>; -/// Open every generation from the rows under this node's W. `Ok(None)` -/// means the newest doesn't open under W (locked). -fn unlock_rows( - w: &WrapKey, - rows: &[at_rest_key::Model], -) -> Result)>, AtRestError> { - check_contiguous(rows)?; - let newest = rows.last().expect("check_contiguous: non-empty"); - let newest_id = ark_id_of(newest)?; - let Ok(ark) = open_ark( - w, - &newest.sealed, - &newest_id, - newest.generation as u32, - &newest.w_label, - ) else { - return Ok(None); - }; - keys_from_newest(rows, ark, Some(w)).map(Some) +fn candidates(db: &[at_rest_key::Model], mirrored: Vec) -> Candidates { + let mut out = Candidates::new(); + for row in db { + out.entry(row.generation as u32) + .or_default() + .push(Candidate { + row: row.clone(), + from_db: true, + }); + } + for row in mirrored { + let list = out.entry(row.generation as u32).or_default(); + if !list.iter().any(|c| c.row == row) { + list.push(Candidate { + row, + from_db: false, + }); + } + } + out } -/// Walk down from the newest ARK. Every older generation must open **by its -/// chain**: that is the only path recovery has, so a broken chain is never -/// accepted silently. +/// Walk down from an authenticated ARK at `top`. For every older generation, +/// pick the candidate whose chain opens under the newer ARK. That is the +/// only path recovery has, so nothing is accepted because W alone opens it. /// -/// With `repair_w` (boot under the root that sealed the rows), a chain that -/// is missing or doesn't open is rebuilt if the generation opens under W, -/// and returned for the caller to store. Without it (recovery under a new -/// root), a broken chain is an error. -fn keys_from_newest( - rows: &[at_rest_key::Model], - newest_ark: Ark, +/// With `repair_w` (boot under the root that sealed the rows), a generation +/// with no working chain is accepted if it opens under W, and its chain is +/// rebuilt (logged). Without it (recovery under a new root), that is an +/// error. +/// +/// Returns the rows for 1..=top as they must be stored, and the keys. +fn resolve( + cands: &Candidates, + top: &at_rest_key::Model, + top_ark: Ark, repair_w: Option<&WrapKey>, -) -> Result<(Keys, Vec), AtRestError> { - check_contiguous(rows)?; +) -> Result<(Vec, Keys), AtRestError> { + let top_g = u32::try_from(top.generation) + .map_err(|_| AtRestError::Inconsistent("ARK generation out of range"))?; + let top_id = ark_id_of(top)?; + let mut chosen = vec![top.clone()]; let mut gens = BTreeMap::new(); - let mut rechain = Vec::new(); - let mut newer: Option<(Ark, ArkId)> = None; - for row in rows.iter().rev() { - let id = ark_id_of(row)?; - let g = row.generation as u32; - let this = match &newer { - None => newest_ark.clone(), - Some((n_ark, n_id)) => { - let by_chain = row - .chain - .as_deref() - .and_then(|c| open_chain(n_ark, n_id, c, &id, g).ok()); - match (by_chain, repair_w) { - (Some(ark), _) => ark, - (None, Some(w)) => { - let ark = open_ark(w, &row.sealed, &id, g, &row.w_label).map_err(|_| { - AtRestError::Inconsistent( - "an older ARK generation opens neither by its chain nor under W", - ) - })?; - rechain.push((g, chain_ark(n_ark, n_id, &ark, &id, g))); - ark - } - (None, None) => { - return Err(AtRestError::Inconsistent( - "an older ARK generation's chain is missing or does not open", - )) - } - } + gens.insert( + top_g, + Generation { + ark_id: top_id, + kek: Kek::derive(&top_ark), + ark: top_ark.clone(), + }, + ); + let mut newer = (top_ark, top_id); + for g in (1..top_g).rev() { + let list = cands.get(&g).map(Vec::as_slice).unwrap_or_default(); + let by_chain = list.iter().find_map(|c| { + let id = ark_id_of(&c.row).ok()?; + let chain = c.row.chain.as_deref()?; + let ark = open_chain(&newer.0, &newer.1, chain, &id, g).ok()?; + Some((c.row.clone(), ark, id)) + }); + let found = match (by_chain, repair_w) { + (Some(found), _) => found, + (None, Some(w)) => list + .iter() + .find_map(|c| { + let id = ark_id_of(&c.row).ok()?; + let ark = open_ark(w, &c.row.sealed, &id, g, &c.row.w_label).ok()?; + let mut row = c.row.clone(); + row.chain = Some(chain_ark(&newer.0, &newer.1, &ark, &id, g)); + tracing::warn!( + generation = g, + "at-rest ARK chain was missing or did not open; rebuilt it from the newer generation" + ); + Some((row, ark, id)) + }) + .ok_or(AtRestError::Inconsistent( + "an older ARK generation opens neither by its chain nor under W", + ))?, + (None, None) => { + return Err(AtRestError::Inconsistent( + "an older ARK generation's chain is missing or does not open", + )) } }; - newer = Some((this.clone(), id)); + let (row, ark, id) = found; gens.insert( g, Generation { ark_id: id, - kek: Kek::derive(&this), - ark: this, + kek: Kek::derive(&ark), + ark: ark.clone(), }, ); + chosen.push(row); + newer = (ark, id); } - let ark_1 = &gens.get(&1).expect("check_contiguous: generation 1").ark; + chosen.reverse(); + let ark_1 = &gens.get(&1).expect("the walk ends at generation 1").ark; let keys = Keys { nk: object_name_key(ark_1), dek_tag: DekTagKey::derive(ark_1), gens, }; - Ok((keys, rechain)) + Ok((chosen, keys)) +} + +/// Unlock under this root's W: the newest generation (at least the +/// database's newest) whose row opens under W and whose chain reaches +/// generation 1. Mirrored generations that don't authenticate or don't +/// connect are skipped with a warning. `Ok(None)`: nothing opens (locked). +fn unlock_under_w( + w: &WrapKey, + db_newest: u32, + cands: &Candidates, +) -> Result, Keys)>, AtRestError> { + for (&g, list) in cands.iter().rev() { + if g < db_newest.max(1) { + break; + } + for c in list { + let Ok(id) = ark_id_of(&c.row) else { continue }; + let Ok(ark) = open_ark(w, &c.row.sealed, &id, g, &c.row.w_label) else { + continue; + }; + match resolve(cands, &c.row, ark, Some(w)) { + Ok(found) => return Ok(Some(found)), + // The database's own newest row: a broken store, fail loudly. + Err(e) if c.from_db => return Err(e), + Err(e) => tracing::warn!( + generation = g, + error = %e, + "ignoring a mirrored ARK generation whose chain does not reach generation 1" + ), + } + } + if g > db_newest { + tracing::warn!( + generation = g, + "ignoring a mirrored ARK generation the database lacks: it does not authenticate under this node's root" + ); + } + } + Ok(None) +} + +/// Store what `resolve` chose: insert generations the database lacks, and +/// update the seal or chain of those it has. Every row was authenticated. +async fn persist_chosen( + store: &AtRestStore, + db: &[at_rest_key::Model], + chosen: &[at_rest_key::Model], +) -> Result<(), AtRestError> { + for row in chosen { + match db.iter().find(|d| d.generation == row.generation) { + None => { + tracing::warn!( + generation = row.generation, + "at-rest ARK row was missing from the database; restored an authenticated copy from the mirror" + ); + store.insert_key(row.clone()).await?; + } + Some(d) if d.ark_id == row.ark_id => { + if d.sealed != row.sealed || d.w_label != row.w_label { + store + .reseal_key(row.generation, &row.w_label, row.sealed.clone()) + .await?; + } + if d.chain != row.chain { + if let Some(chain) = &row.chain { + store.set_chain(row.generation, chain.clone()).await?; + } + } + } + Some(_) => { + return Err(AtRestError::Inconsistent( + "the database holds a different ARK for a generation the authenticated chain covers", + )) + } + } + } + Ok(()) +} + +/// Mirrored ARK rows, unauthenticated. Junk objects (not JSON, a newer +/// format, bad fields) are skipped with a warning; only mirror I/O fails. +async fn mirrored_rows(mirror: &dyn KeyMirror) -> Result, AtRestError> { + let mut names = mirror.list(ARK_MIRROR_PREFIX).await.map_err(mirror_err)?; + names.sort(); + let mut rows = Vec::new(); + for name in names { + let Some(bytes) = get_capped(mirror, &name).await.map_err(mirror_err)? else { + continue; + }; + let parsed = serde_json::from_slice::(&bytes) + .map_err(|e| e.to_string()) + .and_then(|m| m.into_row().map_err(|e| e.to_string())); + match parsed { + Ok(row) if row.generation >= 1 && row.ark_id.len() == 16 => rows.push(row), + Ok(_) => { + tracing::warn!(object = ?name, "ignoring an at-rest mirror object with bad fields") + } + Err(error) => { + tracing::warn!(object = ?name, %error, "ignoring an unreadable at-rest mirror object") + } + } + } + Ok(rows) +} + +/// The generation a locked node waits for. The pinned bundle names it when +/// one is found (the pin is attested config); otherwise the database's +/// newest; otherwise (empty database) the mirror's only contiguous run. +fn locked_target( + db: &[at_rest_key::Model], + cands: &Candidates, + pinned: Option<&EscrowBundle>, +) -> Result<(u32, ArkId), AtRestError> { + if let Some(b) = pinned { + let known = cands + .get(&b.generation) + .is_some_and(|l| l.iter().any(|c| c.row.ark_id == b.ark_id.0)); + if known && b.generation as usize >= db.len() { + return Ok((b.generation, b.ark_id)); + } + } + if let Some(newest) = db.last() { + return Ok((newest.generation as u32, ark_id_of(newest)?)); + } + let mut target = None; + for g in 1.. { + match cands.get(&g).map(Vec::as_slice) { + Some([only]) => target = Some((g, ark_id_of(&only.row)?)), + Some([]) | None => break, + Some(_) => return Err(AtRestError::Inconsistent( + "the mirror holds conflicting ARK rows and no pinned escrow names the right one", + )), + } + } + target.ok_or(AtRestError::Inconsistent( + "sealed at-rest material exists but no ARK row or mirrored ARK was found", + )) } pub struct AtRestKeyring { @@ -358,29 +538,57 @@ pub struct AtRestKeyring { impl AtRestKeyring { /// Load the keyring at boot. /// - /// - Mirrored ARK generations missing from the database (a restore to - /// an earlier point) are put back first. - /// - Rows open under this root's W: unlocked. A chain that is missing or + /// - The newest generation that opens under this root's W, with a chain + /// down to generation 1, unlocks the keys. Mirrored generations the + /// database lacks (a restore to an earlier point) are stored only once + /// they authenticate this way; anything else in the mirror (forged, + /// junk, gapped) is skipped with a warning. A chain that is missing or /// doesn't open is rebuilt and logged. - /// - Rows exist but don't open: **locked** (data intact; recover with two - /// shares). - /// - No rows: provision generation 1 if `config.provision`, else not - /// provisioned. Never when any sealed material exists without an ARK - /// (data keys, escrow rows, any mirror object), and never after a - /// store or mirror error. + /// - Rows exist but nothing opens: **locked** (data intact; recover with + /// two shares of the pinned escrow). + /// - No rows anywhere: provision generation 1 if `config.provision`, + /// else not provisioned. Never when any sealed material exists without + /// an ARK (data keys, escrow rows, any mirror object), and never after + /// a store or mirror error. pub async fn open( store: AtRestStore, mirror: Arc, root: &[u8], config: KeyringConfig, ) -> Result { - let w = WrapKey::derive(root); - Self::restore_rows(&store, &*mirror).await?; - let mut rows = store.keys().await?; - if rows.is_empty() { - if store.has_space_keys().await? - || store.has_escrow_rows().await? - || !mirror + let keyring = Self { + store, + mirror, + w: WrapKey::derive(root), + config, + phase: RwLock::new(Phase::NotProvisioned), + deks: Mutex::new(LruCache::new( + NonZeroUsize::new(DEK_CACHE).expect("non-zero"), + )), + current: Mutex::new(LruCache::new( + NonZeroUsize::new(DEK_CACHE).expect("non-zero"), + )), + escrow_mirrored: Mutex::new(None), + intake: tokio::sync::Mutex::new(ShareIntake::new()), + dek_create: tokio::sync::Mutex::new(()), + }; + keyring.load().await?; + if keyring.state() == KeyringState::Unlocked { + keyring.ensure_escrow().await?; + } + Ok(keyring) + } + + /// Read the stored and mirrored rows and set the phase. + async fn load(&self) -> Result<(), AtRestError> { + let mut db = self.store.keys().await?; + check_contiguous(&db)?; + let mirrored = mirrored_rows(&*self.mirror).await?; + if db.is_empty() && mirrored.is_empty() { + if self.store.has_space_keys().await? + || self.store.has_escrow_rows().await? + || !self + .mirror .list(MIRROR_PREFIX) .await .map_err(mirror_err)? @@ -388,142 +596,70 @@ impl AtRestKeyring { { // Another process may have provisioned since we read: its // ARK row is written before anything else. - rows = store.keys().await?; - if rows.is_empty() { + db = self.store.keys().await?; + if db.is_empty() { return Err(AtRestError::Inconsistent( "sealed at-rest material exists but no ARK row or mirrored ARK was found", )); } - } else if config.provision { + } else if self.config.provision { let ark = Ark::generate(); let ark_id = ArkId::generate(); - store + self.store .insert_key(at_rest_key::Model { generation: 1, ark_id: ark_id.0.to_vec(), w_label: W_LABEL.into(), - sealed: seal_ark(&w, &ark, &ark_id, 1), + sealed: seal_ark(&self.w, &ark, &ark_id, 1), chain: None, created_at: now_rfc3339(), }) .await?; // Another process may have won the insert; use what's stored. - rows = store.keys().await?; - if rows.first().map(|r| r.ark_id.as_slice()) == Some(&ark_id.0[..]) { + db = self.store.keys().await?; + if db.first().map(|r| r.ark_id.as_slice()) == Some(&ark_id.0[..]) { tracing::info!(generation = 1, %ark_id, "at-rest root key provisioned"); } + } else { + return Ok(()); } + check_contiguous(&db)?; } - let phase = if rows.is_empty() { - Phase::NotProvisioned - } else { - match unlock_rows(&w, &rows)? { - Some((keys, rechain)) => { - if !rechain.is_empty() { - Self::store_rechain(&store, rechain).await?; - rows = store.keys().await?; - } - for row in &rows { - Self::mirror_row(&*mirror, row).await?; - } - Phase::Unlocked(Arc::new(keys)) - } - None => { - let newest = rows.last().expect("non-empty"); - tracing::warn!( - generation = newest.generation, - "at-rest keys locked: this node's root cannot unseal the ARK" - ); - Phase::Locked { - generation: newest.generation as u32, - ark_id: ark_id_of(newest)?, - } + let cands = candidates(&db, mirrored); + if let Some((mut chosen, keys)) = unlock_under_w(&self.w, db.len() as u32, &cands)? { + // A generation the database lacks is stored with a fresh seal + // under W, never with the mirror's bytes. + for row in chosen.iter_mut() { + if !db.iter().any(|d| d.generation == row.generation) { + let g = row.generation as u32; + let gen = &keys.gens[&g]; + row.sealed = seal_ark(&self.w, &gen.ark, &gen.ark_id, g); + row.w_label = W_LABEL.into(); } } - }; - let keyring = Self { - store, - mirror, - w, - config, - phase: RwLock::new(phase), - deks: Mutex::new(LruCache::new( - NonZeroUsize::new(DEK_CACHE).expect("non-zero"), - )), - current: Mutex::new(LruCache::new( - NonZeroUsize::new(DEK_CACHE).expect("non-zero"), - )), - escrow_mirrored: Mutex::new(None), - intake: tokio::sync::Mutex::new(ShareIntake::new()), - dek_create: tokio::sync::Mutex::new(()), - }; - if keyring.state() == KeyringState::Unlocked { - keyring.ensure_escrow().await?; - } - Ok(keyring) - } - - async fn store_rechain(store: &AtRestStore, rechain: Vec) -> Result<(), AtRestError> { - for (g, chain) in rechain { - tracing::warn!( - generation = g, - "at-rest ARK chain was missing or did not open; rebuilt it from the newer generation" - ); - store.set_chain(g as i32, chain).await?; - } - Ok(()) - } - - /// Put back mirrored ARK generations the database lacks, and chains the - /// database lost. Both are AEAD-checked when used, so a bad mirror - /// object can make the node fail closed but never feeds it a key. - async fn restore_rows(store: &AtRestStore, mirror: &dyn KeyMirror) -> Result<(), AtRestError> { - let mut names = mirror.list(ARK_MIRROR_PREFIX).await.map_err(mirror_err)?; - if names.is_empty() { + persist_chosen(&self.store, &db, &chosen).await?; + for row in self.store.keys().await? { + Self::mirror_row(&*self.mirror, &row).await?; + } + *self.phase.write() = Phase::Unlocked(Arc::new(keys)); return Ok(()); } - names.sort(); - let have: BTreeMap = store - .keys() - .await? - .into_iter() - .map(|r| (r.generation, r)) - .collect(); - for name in names { - let bytes = - mirror - .get(&name) - .await - .map_err(mirror_err)? - .ok_or(AtRestError::Inconsistent( - "mirrored ARK vanished while listing", - ))?; - let row = serde_json::from_slice::(&bytes) - .map_err(mirror_err)? - .into_row() - .map_err(mirror_err)?; - match have.get(&row.generation) { - None => { - tracing::warn!( - generation = row.generation, - "at-rest ARK row was missing from the database; restored from the mirror" - ); - store.insert_key(row).await?; - } - Some(db) - if db.chain.is_none() && row.chain.is_some() && db.ark_id == row.ark_id => - { - tracing::warn!( - generation = row.generation, - "at-rest ARK chain was missing from the database; restored from the mirror" - ); - store - .set_chain(row.generation, row.chain.expect("checked")) - .await?; - } - Some(_) => {} + // A stored or mirrored escrow bundle that can't be read must not + // stop the boot: the node then just has no bundle to recover with. + let pinned = match self.pinned_bundle().await { + Ok(b) => b, + Err(e @ (AtRestError::Store(_) | AtRestError::Mirror(_))) => return Err(e), + Err(error) => { + tracing::warn!(%error, "ignoring the pinned escrow bundle: it can't be read"); + None } - } + }; + let (generation, ark_id) = locked_target(&db, &cands, pinned.as_ref())?; + tracing::warn!( + generation, + "at-rest keys locked: this node's root cannot unseal the ARK" + ); + *self.phase.write() = Phase::Locked { generation, ark_id }; Ok(()) } @@ -533,7 +669,12 @@ impl AtRestKeyring { ) -> Result<(), AtRestError> { let name = ark_mirror_name(row.generation as u32, &hex::encode(&row.ark_id)); let body = serde_json::to_vec(&MirroredArk::from_row(row)).expect("serializable"); - if mirror.get(&name).await.map_err(mirror_err)?.as_deref() != Some(body.as_slice()) { + if get_capped(mirror, &name) + .await + .map_err(mirror_err)? + .as_deref() + != Some(body.as_slice()) + { mirror.put(&name, body).await.map_err(mirror_err)?; } Ok(()) @@ -566,8 +707,14 @@ impl AtRestKeyring { } /// Make sure the newest generation is escrowed to exactly the configured - /// holders. A different holder list re-splits the same ARK into a new - /// bundle (new EFP, so a new ceremony); old bundles are kept inactive. + /// holders. + /// + /// Without a pin, a different holder list re-splits the same ARK into a + /// new bundle (new EFP, so a new ceremony); old bundles are kept + /// inactive. With `verified_fp` pinned, the node **never** re-splits: + /// the only bundle it uses is the pinned one (re-activated if the + /// database lost it). Changing holders means removing the pin in the + /// attested compose first. async fn ensure_escrow(&self) -> Result, AtRestError> { if self.config.holders.is_empty() { return Ok(None); @@ -581,6 +728,25 @@ impl AtRestKeyring { return Ok(Some(bundle)); } } + if let Some(pin) = self.config.pinned_efp { + let usable = self.pinned_bundle().await?.filter(|b| { + b.holders() == self.config.holders && b.ark_id == gen.ark_id && b.generation == g + }); + let Some(bundle) = usable else { + return Err(AtRestError::EscrowPinned); + }; + let row = at_rest_escrow::Model { + efp: pin.to_string(), + generation: g as i32, + bundle: serde_json::to_string(&bundle).expect("serializable"), + active: true, + record: None, + created_at: now_rfc3339(), + }; + self.store.activate_escrow(row.clone()).await?; + self.mirror_escrow(&row).await?; + return Ok(Some(bundle)); + } let bundle = EscrowBundle::create( &gen.ark, gen.ark_id, @@ -611,7 +777,11 @@ impl AtRestKeyring { return Ok(()); } let name = escrow_mirror_name(&row.efp); - if self.mirror.get(&name).await.map_err(mirror_err)?.is_none() { + if get_capped(&*self.mirror, &name) + .await + .map_err(mirror_err)? + .is_none() + { self.mirror .put(&name, row.bundle.clone().into_bytes()) .await @@ -631,9 +801,7 @@ impl AtRestKeyring { if let Some(row) = self.store.escrow(&efp).await? { return Self::parse_bundle(&row).map(Some); } - let Some(bytes) = self - .mirror - .get(&escrow_mirror_name(&efp)) + let Some(bytes) = get_capped(&*self.mirror, &escrow_mirror_name(&efp)) .await .map_err(mirror_err)? else { @@ -791,16 +959,18 @@ impl AtRestKeyring { /// Take one wrapped share (age file to this boot's recovery recipient). /// /// Each share is checked against its commitment before it occupies one of - /// three slots (one per index); submissions are rate limited per boot. + /// three slots (one per index). Only shares that pass (decrypt and match + /// a commitment) spend this boot's submission budget, so junk can't + /// starve genuine holders; the HTTP layer limits failures per client. /// Share bytes are never logged. pub async fn submit_share(&self, wrapped: &[u8]) -> Result { let mut intake = self.intake.lock().await; - if !intake.take_token() { - return Err(ShareRefusal::RateLimited); - } let share = intake.identity.open(wrapped)?; let bundle = self.intake_bundle().await?; bundle.verify_share(&share)?; + if !intake.take_token() { + return Err(ShareRefusal::RateLimited); + } let efp = bundle.fingerprint(); if intake.efp != Some(efp) { intake.clear(); @@ -863,28 +1033,35 @@ impl AtRestKeyring { Ok(outcome) } - /// Locked โ†’ unlocked with a recovered newest ARK: re-seal every - /// generation under this node's W, mirror, and serve keys. + /// Locked โ†’ unlocked with a recovered ARK: authenticate every older + /// generation by its chain (database rows first, then mirrored ones), + /// re-seal all of them under this node's W, store, mirror, serve keys. async fn unlock_with(&self, ark: Ark, bundle: &EscrowBundle) -> Result<(), AtRestError> { - let rows = self.store.keys().await?; - let newest = rows.last().ok_or(AtRestError::NotProvisioned)?; - if newest.generation as u32 != bundle.generation || ark_id_of(newest)? != bundle.ark_id { + let db = self.store.keys().await?; + check_contiguous(&db)?; + if (bundle.generation as usize) < db.len() { return Err(AtRestError::Inconsistent( "recovered ARK is not the newest generation", )); } + let cands = candidates(&db, mirrored_rows(&*self.mirror).await?); + let top = cands + .get(&bundle.generation) + .and_then(|l| l.iter().find(|c| c.row.ark_id == bundle.ark_id.0)) + .map(|c| c.row.clone()) + .ok_or(AtRestError::Inconsistent( + "no stored or mirrored row for the recovered generation", + ))?; // Under a new root the rows still carry the old W: every older // generation must open by its chain, or recovery fails loudly. - let (keys, _) = keys_from_newest(&rows, ark, None)?; - for (g, gen) in &keys.gens { - self.store - .reseal_key( - *g as i32, - W_LABEL, - seal_ark(&self.w, &gen.ark, &gen.ark_id, *g), - ) - .await?; + let (mut chosen, keys) = resolve(&cands, &top, ark, None)?; + for row in chosen.iter_mut() { + let g = row.generation as u32; + let gen = &keys.gens[&g]; + row.sealed = seal_ark(&self.w, &gen.ark, &gen.ark_id, g); + row.w_label = W_LABEL.into(); } + persist_chosen(&self.store, &db, &chosen).await?; for row in self.store.keys().await? { Self::mirror_row(&*self.mirror, &row).await?; } @@ -924,15 +1101,12 @@ impl AtRestKeyring { chain_ark(&ark, &ark_id, &old.ark, &old.ark_id, g), ) .await?; - let rows = self.store.keys().await?; - let (keys, rechain) = unlock_rows(&self.w, &rows)?.ok_or(AtRestError::Inconsistent( - "the stored newest generation does not open under this node's root", - ))?; - Self::store_rechain(&self.store, rechain).await?; - for row in &self.store.keys().await? { - Self::mirror_row(&*self.mirror, row).await?; + self.load().await?; + if self.state() != KeyringState::Unlocked { + return Err(AtRestError::Inconsistent( + "the stored newest generation does not open under this node's root", + )); } - *self.phase.write() = Phase::Unlocked(Arc::new(keys)); self.current.lock().clear(); self.ensure_escrow().await?; if !won { @@ -944,7 +1118,36 @@ impl AtRestKeyring { Ok(next) } - /// A space's DEK row from the mirror, put back into the database. + /// A mirrored DEK row for `space`, only if its wrap authenticates under + /// the stated generation's KEK for exactly this space and id. + fn authentic_mirrored_dek( + keys: &Keys, + space: &str, + dek_id: u32, + bytes: &[u8], + ) -> Option<(space_data_key::Model, DataKey)> { + let row = serde_json::from_slice::(bytes) + .ok()? + .into_row(space) + .ok()?; + if row.dek_id as u32 != dek_id { + return None; + } + let gen = keys.gens.get(&(row.generation as u32))?; + let key = unwrap_dek( + &gen.kek, + &row.wrapped, + space, + DekId::new(dek_id), + row.generation as u32, + &gen.ark_id, + ) + .ok()?; + Some((row, key)) + } + + /// A space's DEK row from the mirror, put back into the database once + /// it authenticates. Anything else in the mirror is ignored (warned). async fn restore_dek( &self, keys: &Keys, @@ -952,23 +1155,19 @@ impl AtRestKeyring { dek_id: u32, ) -> Result, DataKeyError> { let name = dek_mirror_name(&keys.dek_tag.tag(space), dek_id); - let Some(bytes) = self - .mirror - .get(&name) + let Some(bytes) = get_capped(&*self.mirror, &name) .await .map_err(|e| backend(mirror_err(e)))? else { return Ok(None); }; - let row = serde_json::from_slice::(&bytes) - .map_err(|e| backend(mirror_err(e)))? - .into_row(space) - .map_err(|e| backend(mirror_err(e)))?; - if row.dek_id as u32 != dek_id { - return Err(backend(AtRestError::Inconsistent( - "mirrored data key has a different id than its name", - ))); - } + let Some((row, _)) = Self::authentic_mirrored_dek(keys, space, dek_id, &bytes) else { + tracing::warn!( + dek_id, + "ignoring a mirrored data key that does not authenticate for its space" + ); + return Ok(None); + }; tracing::warn!( dek_id, "at-rest data key was missing from the database; restored from the mirror" @@ -981,45 +1180,41 @@ impl AtRestKeyring { } /// Make sure the mirror holds exactly this DEK row before any object is - /// sealed under it. A different mirrored row with the same id means - /// objects may exist under another key: refuse, never overwrite. + /// sealed under it. A *genuine* different DEK under the same id means + /// objects may exist under another key: refuse, never overwrite. A + /// mirror object that doesn't authenticate carries no key and is + /// replaced. async fn mirror_dek( &self, keys: &Keys, row: &space_data_key::Model, ) -> Result<(), DataKeyError> { - let name = dek_mirror_name(&keys.dek_tag.tag(&row.space), row.dek_id as u32); - match self - .mirror - .get(&name) + let dek_id = row.dek_id as u32; + let name = dek_mirror_name(&keys.dek_tag.tag(&row.space), dek_id); + let existing = get_capped(&*self.mirror, &name) .await - .map_err(|e| backend(mirror_err(e)))? - { - None => { - let body = serde_json::to_vec(&MirroredDek::from_row(row)).expect("serializable"); - self.mirror - .put(&name, body) - .await - .map_err(|e| backend(mirror_err(e))) - } - Some(bytes) => { - let same = serde_json::from_slice::(&bytes) - .ok() - .and_then(|m| m.into_row(&row.space).ok()) - .is_some_and(|m| { - m.dek_id == row.dek_id - && m.generation == row.generation - && m.wrapped == row.wrapped - }); - if same { - Ok(()) - } else { - Err(backend(AtRestError::Inconsistent( + .map_err(|e| backend(mirror_err(e)))?; + if let Some(bytes) = existing { + match Self::authentic_mirrored_dek(keys, &row.space, dek_id, &bytes) { + Some((m, _)) if m.generation == row.generation && m.wrapped == row.wrapped => { + return Ok(()) + } + Some(_) => { + return Err(backend(AtRestError::Inconsistent( "the mirror holds a different data key with this id", ))) } + None => tracing::warn!( + dek_id, + "replacing a mirrored data key that does not authenticate for its space" + ), } } + let body = serde_json::to_vec(&MirroredDek::from_row(row)).expect("serializable"); + self.mirror + .put(&name, body) + .await + .map_err(|e| backend(mirror_err(e))) } /// Generate, store and return the space's first DEK row. If another diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index 81b6321b..716b1ffd 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -1,6 +1,7 @@ use std::io::Read; use std::sync::Arc; +use base64::Engine as _; use sea_orm::{ ColumnTrait, ConnectOptions, Database, DatabaseConnection, EntityTrait, QueryFilter, }; @@ -8,7 +9,7 @@ use sea_orm_migration::MigratorTrait; use tinycloud_at_rest_escrow::{wrap_share, EscrowBundle, HolderRecipient, SharePayload}; use super::*; -use crate::at_rest::store::MemoryMirror; +use crate::at_rest::store::{MemoryMirror, MAX_MIRROR_OBJECT}; use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; const ROOT: [u8; 64] = [7; 64]; @@ -433,17 +434,17 @@ async fn gate_requires_pinned_verified_escrow_for_the_configured_recipients() { assert_eq!(k.gate(false).await, Ok(GatePass::Verified { efp })); drop(k); - // New recipients in config re-split the same ARK: a new bundle, and the - // old pin no longer opens the gate. + // New recipients in config with the old pin removed re-split the same + // ARK: a new bundle that the old pin doesn't match. let changed = holders(); - let k = open(&db, &mirror, &ROOT, config(&changed, Some(efp))) + let k = open(&db, &mirror, &ROOT, config(&changed, None)) .await .unwrap(); let new_efp = k.escrow_view().await.unwrap().unwrap().efp; assert_ne!(new_efp, efp); assert!(matches!( k.gate(false).await, - Err(GateRefusal::PinMismatch { .. }) + Err(GateRefusal::NotPinned { .. }) )); let bundles = at_rest_escrow::Entity::find().all(&db).await.unwrap(); assert_eq!(bundles.len(), 2); @@ -540,20 +541,39 @@ async fn rotation_keeps_old_generations_and_survives_recovery() { } #[tokio::test] -async fn share_intake_is_rate_limited_per_boot() { +async fn junk_never_spends_the_share_budget() { let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); - for _ in 0..SHARE_BURST { - assert!(!matches!( + // Any amount of junk, or shares of another escrow, is refused but free. + let foreign = EscrowBundle::create( + &Ark::generate(), + ArkId::generate(), + 1, + &h.recipients, + RecipientPolicy::AllowSoftware, + ) + .unwrap(); + let recipient = k.escrow_view().await.unwrap().unwrap().recovery_recipient; + for _ in 0..3 * SHARE_BURST { + assert!(matches!( k.submit_share(b"junk").await, - Err(ShareRefusal::RateLimited) + Err(ShareRefusal::Invalid(_)) + )); + let wrapped = wrap_share(&recipient, &h.decrypt(&foreign, 0)).unwrap(); + assert!(matches!( + k.submit_share(wrapped.as_bytes()).await, + Err(ShareRefusal::Invalid(_)) )); } + // A genuine share still goes through after the junk... assert!(matches!( - k.submit_share(b"junk").await, - Err(ShareRefusal::RateLimited) + submit(&k, &h, 0).await, + Ok(ShareOutcome::Held { .. }) )); - // Valid shares are throttled the same way. + // ...and only genuine submissions count against the per-boot budget. + for _ in 1..SHARE_BURST { + submit(&k, &h, 0).await.unwrap(); + } assert!(matches!( submit(&k, &h, 0).await, Err(ShareRefusal::RateLimited) @@ -901,27 +921,207 @@ async fn escrow_rows_or_any_mirror_object_block_provisioning() { .is_empty()); } +/// Plant an unauthenticated ARK row in the mirror, as a bucket writer could. +async fn plant_ark(mirror: &MemoryMirror, generation: u32) { + let ark_id = ArkId::generate(); + let forged = MirroredArk { + version: 1, + generation, + ark_id: hex::encode(ark_id.0), + w_label: W_LABEL.into(), + sealed: base64::engine::general_purpose::STANDARD.encode([0x41; 60]), + chain: Some(base64::engine::general_purpose::STANDARD.encode([0x42; 60])), + created_at: now_rfc3339(), + }; + mirror + .put( + &ark_mirror_name(generation, &hex::encode(ark_id.0)), + serde_json::to_vec(&forged).unwrap(), + ) + .await + .unwrap(); +} + +/// Junk the parser must skip: not JSON, a newer format, and oversized. +async fn plant_junk(mirror: &MemoryMirror) { + mirror + .put(&format!("{ARK_MIRROR_PREFIX}zzz.json"), b"".to_vec()) + .await + .unwrap(); + let newer = serde_json::json!({"version": 2, "generation": 1}); + mirror + .put( + &ark_mirror_name(1, &"f".repeat(32)), + serde_json::to_vec(&newer).unwrap(), + ) + .await + .unwrap(); + mirror + .put( + &ark_mirror_name(1, &"e".repeat(32)), + vec![b' '; MAX_MIRROR_OBJECT + 1], + ) + .await + .unwrap(); +} + #[tokio::test] -async fn a_newer_mirror_format_is_refused_not_guessed() { +async fn forged_gapped_or_junk_mirror_rows_never_lock_or_stop_a_healthy_node() { let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); - open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let dek = k.current_key("space-a").await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + plant_ark(&mirror, 2).await; // forged next generation + plant_ark(&mirror, 4).await; // gap: no generation 3 + plant_junk(&mirror).await; + + for _ in 0..2 { + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + assert_eq!(k.current_key("space-a").await.unwrap().bytes(), dek.bytes()); + let gens: Vec = AtRestStore::new(db.clone()) + .keys() + .await + .unwrap() + .iter() + .map(|r| r.generation) + .collect(); + assert_eq!(gens, vec![1], "nothing unauthenticated was stored"); + } + + // A new root still waits for the real generation and recovers with it. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert_eq!(k.state(), KeyringState::Locked); + submit(&k, &h, 0).await.unwrap(); + assert_eq!(submit(&k, &h, 1).await.unwrap(), ShareOutcome::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + dek.bytes() + ); +} + +#[tokio::test] +async fn an_empty_database_takes_back_only_authenticated_mirror_rows() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let dek = k.current_key("space-a").await.unwrap(); + k.rotate_ark().await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + plant_ark(&mirror, 3).await; + plant_junk(&mirror).await; + + // A restore to before any key row, same root: generations 1 and 2 come + // back (they authenticate); the forged generation 3 doesn't. at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); - let name = mirror - .names() - .into_iter() - .find(|n| n.starts_with(ARK_MIRROR_PREFIX)) + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + dek.bytes() + ); + assert_eq!(at_rest_key::Entity::find().all(&db).await.unwrap().len(), 2); + drop(k); + + // The same restore under a new root: nothing authenticates until the + // pinned escrow's shares do; then the rows are stored, resealed. + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await .unwrap(); - let mut obj: serde_json::Value = - serde_json::from_slice(&mirror.get(&name).await.unwrap().unwrap()).unwrap(); - obj["version"] = serde_json::json!(2); - mirror - .put(&name, serde_json::to_vec(&obj).unwrap()) + assert_eq!(k.state(), KeyringState::Locked); + assert!(at_rest_key::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); + submit(&k, &h, 2).await.unwrap(); + assert_eq!(submit(&k, &h, 0).await.unwrap(), ShareOutcome::Unlocked); + assert_eq!( + k.key("space-a", DekId::new(1)).await.unwrap().bytes(), + dek.bytes() + ); + assert_eq!(at_rest_key::Entity::find().all(&db).await.unwrap().len(), 2); + drop(k); + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) .await .unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + + // Only junk and no rows at all: fail closed, never provision over it. + let (db, mirror) = (self::db().await, Arc::new(MemoryMirror::default())); + plant_junk(&mirror).await; assert!(matches!( open(&db, &mirror, &ROOT, config(&h, None)).await, - Err(AtRestError::Mirror(e)) if e.contains("version 2") + Err(AtRestError::Inconsistent(_)) + )); + assert!(at_rest_key::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn a_mirrored_data_key_for_another_space_is_never_stored() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let a = k.current_key("space-a").await.unwrap(); + // A bucket writer copies space-a's mirrored DEK to space-b's name. + let keys = k.keys().unwrap(); + let a_name = dek_mirror_name(&keys.dek_tag.tag("space-a"), 1); + let b_name = dek_mirror_name(&keys.dek_tag.tag("space-b"), 1); + let copied = mirror.get(&a_name).await.unwrap().unwrap(); + mirror.put(&b_name, copied.clone()).await.unwrap(); + + // It doesn't authenticate for space-b: not stored, not served, replaced. + assert!(matches!( + k.key("space-b", DekId::new(1)).await, + Err(DataKeyError::UnknownKey { .. }) + )); + assert!(AtRestStore::new(db.clone()) + .dek("space-b", 1) + .await + .unwrap() + .is_none()); + let b = k.current_key("space-b").await.unwrap(); + assert_ne!(b.bytes(), a.bytes()); + assert_ne!(mirror.get(&b_name).await.unwrap().unwrap(), copied); +} + +#[tokio::test] +async fn a_pinned_escrow_is_never_re_split() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + // Different recipients with the pin still set: refused, nothing written. + let changed = holders(); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&changed, Some(efp))).await, + Err(AtRestError::EscrowPinned) )); + assert_eq!( + at_rest_escrow::Entity::find().all(&db).await.unwrap().len(), + 1 + ); + // Pinned and the database lost the bundle: the pinned one comes back + // from the mirror, no new split. + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert_eq!(k.escrow_view().await.unwrap().unwrap().efp, efp); + assert_eq!( + at_rest_escrow::Entity::find().all(&db).await.unwrap().len(), + 1 + ); } #[tokio::test] diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs index 0d23466d..d7192f1c 100644 --- a/tinycloud-core/src/at_rest/store.rs +++ b/tinycloud-core/src/at_rest/store.rs @@ -21,6 +21,16 @@ use crate::RecoveringConnection; pub type MirrorError = Box; +/// The largest mirror object a [`KeyMirror`] reads. Key objects are a few +/// hundred bytes; a larger one is junk and is never read into memory. +pub const MAX_MIRROR_OBJECT: usize = 64 * 1024; + +/// A mirror object over [`MAX_MIRROR_OBJECT`]. The keyring skips it like +/// any other unreadable object instead of failing. +#[derive(Debug, thiserror::Error)] +#[error("mirror object is larger than {MAX_MIRROR_OBJECT} bytes")] +pub struct MirrorObjectTooLarge; + /// Named control objects next to the node's blocks (`_tc/at-rest/โ€ฆ`). The S3 /// and filesystem block stores implement it in the server crate. #[async_trait::async_trait] @@ -447,7 +457,10 @@ impl KeyMirror for MemoryMirror { async fn get(&self, name: &str) -> Result>, MirrorError> { self.check()?; - Ok(self.objects.lock().get(name).cloned()) + match self.objects.lock().get(name) { + Some(bytes) if bytes.len() > MAX_MIRROR_OBJECT => Err(Box::new(MirrorObjectTooLarge)), + other => Ok(other.cloned()), + } } async fn list(&self, prefix: &str) -> Result, MirrorError> { From 4eb6ebc9dce99de50045c478d533a4f69821100b Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 11:16:12 +0000 Subject: [PATCH 06/15] fix(TC-887): escrow is established only at boot; read paths are read-only (#326) - establish_escrow runs at boot, after a rotation and after a recovery. An active bundle for other holders refuses the boot (EscrowHoldersChanged): the node never re-splits on its own. With no active bundle, a pinned bundle is re-activated from the database or mirror (the pin authenticates it); unpinned, a mirrored bundle whose mirrored record verifies under the ARK is restored; otherwise a new one is made. A missing record is restored from the mirror when its MAC verifies. A database restore no longer forces a new ceremony. - escrow_view, verification_record, gate and share intake only read the active bundle; no request writes escrow rows or mirror objects. - The newest generation never keeps a chain: boot clears a stale one, and rotate_ark answers a stale chain with an explicit Inconsistent ("retry"), not a permanent Conflict. - AtRestError::is_transient and reason(): stable, non-secret reason codes for API answers. - Tests: two nodes with different holders (no flip, reads write nothing), restore before the record and before the escrow (pinned and unpinned), a forged mirrored record, a stale newest chain. --- tinycloud-core/src/at_rest/keyring.rs | 290 +++++++++++++++----- tinycloud-core/src/at_rest/keyring/tests.rs | 152 +++++++++- tinycloud-core/src/at_rest/store.rs | 19 ++ 3 files changed, 385 insertions(+), 76 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index 77cbb1f3..2071fa0d 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -22,8 +22,8 @@ use super::hierarchy::{ Kek, WrapKey, W_LABEL, }; use super::store::{ - ark_mirror_name, dek_mirror_name, escrow_mirror_name, AtRestStore, KeyMirror, - MirrorObjectTooLarge, MirroredArk, MirroredDek, ARK_MIRROR_PREFIX, MIRROR_PREFIX, + ark_mirror_name, dek_mirror_name, escrow_mirror_name, record_mirror_name, AtRestStore, + KeyMirror, MirrorObjectTooLarge, MirroredArk, MirroredDek, ARK_MIRROR_PREFIX, MIRROR_PREFIX, }; use super::{DataKey, DataKeyError, DataKeyProvider, DekId, ObjectNameKey}; use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; @@ -60,8 +60,10 @@ pub enum AtRestError { Inconsistent(&'static str), #[error("at-rest key change lost a race: {0}")] Conflict(&'static str), - #[error("at_rest.escrow.verified_fp pins an escrow that is not for the configured recipients and the newest key; the node never re-splits while a fingerprint is pinned (remove the pin in the attested compose to change holders)")] + #[error("at_rest.escrow.verified_fp pins an escrow that is not for the configured recipients and the newest key; the node never re-splits while a fingerprint is pinned")] EscrowPinned, + #[error("the configured escrow recipients differ from the active escrow bundle's; the node never re-splits on its own (an explicit re-escrow is needed)")] + EscrowHoldersChanged, #[error("at-rest escrow: {0}")] Escrow(#[from] EscrowError), #[error("at-rest keys are locked")] @@ -70,6 +72,38 @@ pub enum AtRestError { NotProvisioned, } +impl AtRestError { + /// Whether retrying may help: the store or mirror was unreachable, the + /// keys are locked, or a concurrent change won. Everything else needs an + /// operator. + pub fn is_transient(&self) -> bool { + matches!( + self, + Self::Store(_) + | Self::Mirror(_) + | Self::Conflict(_) + | Self::Locked + | Self::NotProvisioned + ) + } + + /// A stable reason for API answers: a code plus, where it's static and + /// key-free, the explanation. Backend details (SQL, S3) stay in the log. + pub fn reason(&self) -> String { + match self { + Self::Store(_) => "store_unavailable".into(), + Self::Mirror(_) => "mirror_unavailable".into(), + Self::Inconsistent(why) => format!("inconsistent: {why}"), + Self::Conflict(why) => format!("conflict: {why}"), + Self::EscrowPinned => "escrow_pinned".into(), + Self::EscrowHoldersChanged => "escrow_holders_changed".into(), + Self::Escrow(e) => format!("escrow: {e}"), + Self::Locked => "locked".into(), + Self::NotProvisioned => "not_provisioned".into(), + } + } +} + /// Why `encrypt` may not be enabled. Every variant is an operator action. #[derive(Debug, PartialEq, Eq, thiserror::Error)] pub enum GateRefusal { @@ -318,7 +352,11 @@ fn resolve( let top_g = u32::try_from(top.generation) .map_err(|_| AtRestError::Inconsistent("ARK generation out of range"))?; let top_id = ark_id_of(top)?; - let mut chosen = vec![top.clone()]; + // The newest generation has no successor, so it never carries a chain. + let mut chosen = vec![at_rest_key::Model { + chain: None, + ..top.clone() + }]; let mut gens = BTreeMap::new(); gens.insert( top_g, @@ -445,8 +483,9 @@ async fn persist_chosen( .await?; } if d.chain != row.chain { - if let Some(chain) = &row.chain { - store.set_chain(row.generation, chain.clone()).await?; + match &row.chain { + Some(chain) => store.set_chain(row.generation, chain.clone()).await?, + None => store.clear_chain(row.generation).await?, } } } @@ -574,7 +613,7 @@ impl AtRestKeyring { }; keyring.load().await?; if keyring.state() == KeyringState::Unlocked { - keyring.ensure_escrow().await?; + keyring.establish_escrow().await?; } Ok(keyring) } @@ -706,54 +745,71 @@ impl AtRestKeyring { Ok(bundle) } - /// Make sure the newest generation is escrowed to exactly the configured - /// holders. + /// At boot (and after a rotation or recovery): make sure the newest + /// generation has an active escrow bundle for exactly the configured + /// holders. This is the only place a bundle is ever created or + /// activated; request paths only read ([`Self::active_bundle`]). + /// + /// - An active bundle for other holders: refused + /// ([`AtRestError::EscrowHoldersChanged`]); the node never re-splits on + /// its own. + /// - None active and `verified_fp` pinned: the pinned bundle, from the + /// database or the mirror (the pin authenticates it), is re-activated; + /// anything else is [`AtRestError::EscrowPinned`]. + /// - None active, no pin: a mirrored bundle whose mirrored ceremony + /// record verifies under the ARK is restored (a database restore + /// doesn't force a new ceremony); otherwise a new bundle is made. /// - /// Without a pin, a different holder list re-splits the same ARK into a - /// new bundle (new EFP, so a new ceremony); old bundles are kept - /// inactive. With `verified_fp` pinned, the node **never** re-splits: - /// the only bundle it uses is the pinned one (re-activated if the - /// database lost it). Changing holders means removing the pin in the - /// attested compose first. - async fn ensure_escrow(&self) -> Result, AtRestError> { + /// A verification record the database lost is restored from the mirror + /// when it verifies under the ARK. + async fn establish_escrow(&self) -> Result<(), AtRestError> { if self.config.holders.is_empty() { - return Ok(None); + return Ok(()); } let keys = self.keys()?; let (g, gen) = keys.newest(); if let Some(row) = self.store.active_escrow(g as i32).await? { let bundle = Self::parse_bundle(&row)?; - if bundle.holders() == self.config.holders && bundle.ark_id == gen.ark_id { - self.mirror_escrow(&row).await?; - return Ok(Some(bundle)); + if bundle.ark_id != gen.ark_id { + return Err(AtRestError::Inconsistent( + "the active escrow bundle is for another ARK", + )); + } + if bundle.holders() != self.config.holders { + return Err(AtRestError::EscrowHoldersChanged); } - } - if let Some(pin) = self.config.pinned_efp { - let usable = self.pinned_bundle().await?.filter(|b| { - b.holders() == self.config.holders && b.ark_id == gen.ark_id && b.generation == g - }); - let Some(bundle) = usable else { - return Err(AtRestError::EscrowPinned); - }; - let row = at_rest_escrow::Model { - efp: pin.to_string(), - generation: g as i32, - bundle: serde_json::to_string(&bundle).expect("serializable"), - active: true, - record: None, - created_at: now_rfc3339(), - }; - self.store.activate_escrow(row.clone()).await?; self.mirror_escrow(&row).await?; - return Ok(Some(bundle)); + if row.record.is_none() { + self.restore_record(&gen.ark, &bundle).await?; + } + return Ok(()); } - let bundle = EscrowBundle::create( - &gen.ark, - gen.ark_id, - g, - &self.config.holders, - self.config.recipient_policy, - )?; + let wanted = |b: &EscrowBundle| { + b.holders() == self.config.holders && b.ark_id == gen.ark_id && b.generation == g + }; + let restored = match self.config.pinned_efp { + Some(_) => match self.pinned_bundle().await?.filter(wanted) { + Some(bundle) => Some(bundle), + None => return Err(AtRestError::EscrowPinned), + }, + None => self + .verified_mirrored_bundle(&gen.ark) + .await? + .filter(wanted), + }; + let (bundle, created) = match restored { + Some(bundle) => (bundle, false), + None => ( + EscrowBundle::create( + &gen.ark, + gen.ark_id, + g, + &self.config.holders, + self.config.recipient_policy, + )?, + true, + ), + }; let row = at_rest_escrow::Model { efp: bundle.fingerprint().to_string(), generation: g as i32, @@ -764,12 +820,100 @@ impl AtRestKeyring { }; self.store.activate_escrow(row.clone()).await?; self.mirror_escrow(&row).await?; - tracing::warn!( - generation = g, - efp = %row.efp, - "at-rest escrow bundle created; the escrow ceremony must verify it before encryption can be enabled" - ); - Ok(Some(bundle)) + if created { + tracing::warn!( + generation = g, + efp = %row.efp, + "at-rest escrow bundle created; the escrow ceremony must verify it before encryption can be enabled" + ); + } else { + tracing::warn!( + generation = g, + efp = %row.efp, + "at-rest escrow bundle was missing from the database; restored an authenticated copy from the mirror" + ); + if self + .store + .escrow(&row.efp) + .await? + .and_then(|r| r.record) + .is_none() + { + self.restore_record(&gen.ark, &bundle).await?; + } + } + Ok(()) + } + + /// A ceremony record for `bundle` from the mirror, stored only if its + /// MAC verifies under the ARK. + async fn restore_record(&self, ark: &Ark, bundle: &EscrowBundle) -> Result<(), AtRestError> { + let efp = bundle.fingerprint().to_string(); + let Some(bytes) = get_capped(&*self.mirror, &record_mirror_name(&efp)) + .await + .map_err(mirror_err)? + else { + return Ok(()); + }; + match VerificationRecord::from_json(&bytes) { + Ok(record) if record.verify(ark, bundle).is_ok() => { + let json = serde_json::to_string(&record).expect("serializable"); + self.store.save_record(&efp, json).await?; + tracing::warn!(%efp, "at-rest ceremony record was missing from the database; restored a verified copy from the mirror"); + } + _ => tracing::warn!(%efp, "ignoring a mirrored ceremony record that does not verify"), + } + Ok(()) + } + + /// An unpinned node after a database restore: a mirrored bundle that a + /// mirrored ceremony record proves (MAC under the ARK) was verified. + async fn verified_mirrored_bundle( + &self, + ark: &Ark, + ) -> Result, AtRestError> { + let names = self + .mirror + .list(&format!("{MIRROR_PREFIX}record-")) + .await + .map_err(mirror_err)?; + for name in names { + let Some(bytes) = get_capped(&*self.mirror, &name).await.map_err(mirror_err)? else { + continue; + }; + let Ok(record) = VerificationRecord::from_json(&bytes) else { + continue; + }; + let efp = record.efp.to_string(); + let Some(bundle_bytes) = get_capped(&*self.mirror, &escrow_mirror_name(&efp)) + .await + .map_err(mirror_err)? + else { + continue; + }; + let Ok(bundle) = EscrowBundle::from_json(&bundle_bytes) else { + continue; + }; + if bundle.fingerprint() == record.efp && record.verify(ark, &bundle).is_ok() { + return Ok(Some(bundle)); + } + } + Ok(None) + } + + /// The newest generation's active bundle. Read-only: request paths never + /// create, activate or mirror anything. + async fn active_bundle(&self) -> Result, AtRestError> { + if self.config.holders.is_empty() { + return Ok(None); + } + let keys = self.keys()?; + let (g, gen) = keys.newest(); + let Some(row) = self.store.active_escrow(g as i32).await? else { + return Ok(None); + }; + let bundle = Self::parse_bundle(&row)?; + Ok((bundle.ark_id == gen.ark_id).then_some(bundle)) } async fn mirror_escrow(&self, row: &at_rest_escrow::Model) -> Result<(), AtRestError> { @@ -833,9 +977,10 @@ impl AtRestKeyring { } Ok(bundle) } - None => self.ensure_escrow().await?.ok_or(ShareRefusal::NoBundle( - "no escrow recipients are configured", - )), + None => self + .active_bundle() + .await? + .ok_or(ShareRefusal::NoBundle("no active escrow bundle")), } } @@ -876,7 +1021,7 @@ impl AtRestKeyring { if self.state() != KeyringState::Unlocked { return Ok(None); } - match self.ensure_escrow().await? { + match self.active_bundle().await? { Some(bundle) => self.verified_record(&bundle).await, None => Ok(None), } @@ -918,9 +1063,13 @@ impl AtRestKeyring { Err(GateRefusal::NoEscrow) }; } - let bundle = match self.ensure_escrow().await { + let bundle = match self.active_bundle().await { Ok(Some(b)) => b, - Ok(None) => return Err(GateRefusal::NoEscrow), + Ok(None) => { + return Err(GateRefusal::Unavailable( + "no active escrow bundle; restart the node to escrow the key".into(), + )) + } Err(e) => return Err(GateRefusal::Unavailable(e.to_string())), }; if bundle.holders() != self.config.holders { @@ -1016,10 +1165,7 @@ impl AtRestKeyring { .await .map_err(AtRestError::from)?; self.mirror - .put( - &format!("{MIRROR_PREFIX}record-{efp}.json"), - json.into_bytes(), - ) + .put(&record_mirror_name(&efp.to_string()), json.into_bytes()) .await .map_err(mirror_err)?; intake.clear(); @@ -1070,6 +1216,11 @@ impl AtRestKeyring { generation = bundle.generation, "at-rest keys recovered from escrow and resealed under this node's root" ); + // The keys are already unlocked; an escrow problem (for example a + // database that lost the bundle row) is reported, not undone. + if let Err(error) = self.establish_escrow().await { + tracing::error!(%error, "at-rest keys recovered, but the escrow could not be re-established"); + } Ok(()) } @@ -1108,12 +1259,21 @@ impl AtRestKeyring { )); } self.current.lock().clear(); - self.ensure_escrow().await?; if !won { - return Err(AtRestError::Conflict( - "another node rotated the ARK first; reloaded the stored generations", - )); + let newest = self.keys()?.newest().0; + return Err(if newest > g { + AtRestError::Conflict( + "another node rotated the ARK first; reloaded the stored generations", + ) + } else { + // The newest generation carried a chain with no successor + // (a restored or garbage row). Reloading cleared it. + AtRestError::Inconsistent( + "the newest ARK generation carried a stale chain; it has been cleared, retry the rotation", + ) + }); } + self.establish_escrow().await?; tracing::warn!(generation = next, %ark_id, "at-rest ARK rotated"); Ok(next) } diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index 716b1ffd..ad9be5a3 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -434,21 +434,136 @@ async fn gate_requires_pinned_verified_escrow_for_the_configured_recipients() { assert_eq!(k.gate(false).await, Ok(GatePass::Verified { efp })); drop(k); - // New recipients in config with the old pin removed re-split the same - // ARK: a new bundle that the old pin doesn't match. + // New recipients in config: the node never re-splits on its own, with + // or without the pin. Nothing is written. let changed = holders(); - let k = open(&db, &mirror, &ROOT, config(&changed, None)) + for pin in [None, Some(efp)] { + assert!(matches!( + open(&db, &mirror, &ROOT, config(&changed, pin)).await, + Err(AtRestError::EscrowHoldersChanged) + )); + } + let bundles = at_rest_escrow::Entity::find().all(&db).await.unwrap(); + assert_eq!(bundles.len(), 1); +} + +#[tokio::test] +async fn two_nodes_with_different_holders_never_flip_the_escrow() { + let (db, mirror) = (db().await, Arc::new(MemoryMirror::default())); + let (ha, hb) = (holders(), holders()); + let a = open(&db, &mirror, &ROOT, config(&ha, None)).await.unwrap(); + let efp = a.escrow_view().await.unwrap().unwrap().efp; + // A second node configured with other holders refuses to boot. + assert!(matches!( + open(&db, &mirror, &ROOT, config(&hb, None)).await, + Err(AtRestError::EscrowHoldersChanged) + )); + // Read paths are read-only: any number of them change nothing. + let escrow_objects = || { + mirror + .names() + .iter() + .filter(|n| n.contains("/escrow-")) + .count() + }; + let before = escrow_objects(); + for _ in 0..10 { + assert_eq!(a.escrow_view().await.unwrap().unwrap().efp, efp); + assert!(a.verification_record().await.unwrap().is_none()); + let _ = a.gate(false).await; + } + assert_eq!( + at_rest_escrow::Entity::find().all(&db).await.unwrap().len(), + 1 + ); + assert_eq!(escrow_objects(), before); +} + +/// Run the ceremony test-unlock on an unlocked keyring. +async fn verify_escrow(k: &AtRestKeyring, h: &Holders) -> Efp { + for i in 0..3 { + submit(k, h, i).await.unwrap(); + } + let view = k.escrow_view().await.unwrap().unwrap(); + assert!(view.verified); + view.efp +} + +#[tokio::test] +async fn a_restore_before_the_escrow_or_record_needs_no_new_ceremony() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let efp = verify_escrow(&k, &h).await; + drop(k); + + // Before the record: it comes back from the mirror (its MAC verifies). + at_rest_escrow::Entity::update_many() + .col_expr( + at_rest_escrow::Column::Record, + sea_orm::sea_query::Expr::value(Option::::None), + ) + .exec(&db) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, Some(efp))) + .await + .unwrap(); + assert_eq!(k.gate(false).await, Ok(GatePass::Verified { efp })); + drop(k); + + // Before the escrow row, pinned and unpinned: the same bundle and its + // record come back; no new split. + for pin in [Some(efp), None] { + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, pin)).await.unwrap(); + let view = k.escrow_view().await.unwrap().unwrap(); + assert_eq!((view.efp, view.verified), (efp, true), "pin {pin:?}"); + assert_eq!( + at_rest_escrow::Entity::find().all(&db).await.unwrap().len(), + 1 + ); + } + + // A forged record in the mirror proves nothing: unpinned, a new bundle. + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + let name = format!("{MIRROR_PREFIX}record-{efp}.json"); + let mut forged: serde_json::Value = + serde_json::from_slice(&mirror.get(&name).await.unwrap().unwrap()).unwrap(); + forged["verified_at"] = serde_json::json!("2030-01-01T00:00:00Z"); + mirror + .put(&name, serde_json::to_vec(&forged).unwrap()) .await .unwrap(); - let new_efp = k.escrow_view().await.unwrap().unwrap().efp; - assert_ne!(new_efp, efp); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let view = k.escrow_view().await.unwrap().unwrap(); + assert_ne!(view.efp, efp); + assert!(!view.verified); +} + +#[tokio::test] +async fn a_stale_chain_on_the_newest_generation_is_cleared_not_a_permanent_conflict() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + drop(k); + set_chain(&db, 1, Some(vec![0x33; 60])).await; + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let rows = AtRestStore::new(db.clone()).keys().await.unwrap(); + assert_eq!(chain_of(&rows, 1), None); + assert_eq!(k.rotate_ark().await.unwrap(), 2); + + // Garbage chained after boot: an explicit, retryable error, then success. + set_chain(&db, 2, Some(vec![0x33; 60])).await; assert!(matches!( - k.gate(false).await, - Err(GateRefusal::NotPinned { .. }) + k.rotate_ark().await, + Err(AtRestError::Inconsistent(why)) if why.contains("retry") )); - let bundles = at_rest_escrow::Entity::find().all(&db).await.unwrap(); - assert_eq!(bundles.len(), 2); - assert_eq!(bundles.iter().filter(|b| b.active).count(), 1); + assert_eq!(k.rotate_ark().await.unwrap(), 3); } #[tokio::test] @@ -1102,12 +1217,27 @@ async fn a_pinned_escrow_is_never_re_split() { let changed = holders(); assert!(matches!( open(&db, &mirror, &ROOT, config(&changed, Some(efp))).await, - Err(AtRestError::EscrowPinned) + Err(AtRestError::EscrowHoldersChanged) )); assert_eq!( at_rest_escrow::Entity::find().all(&db).await.unwrap().len(), 1 ); + // Pinned to a bundle for other holders, with the database's row gone: + // still no split. + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&changed, Some(efp))).await, + Err(AtRestError::EscrowPinned) + )); + assert!(at_rest_escrow::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); // Pinned and the database lost the bundle: the pinned one comes back // from the mirror, no new split. at_rest_escrow::Entity::delete_many() diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs index d7192f1c..60fef976 100644 --- a/tinycloud-core/src/at_rest/store.rs +++ b/tinycloud-core/src/at_rest/store.rs @@ -52,6 +52,10 @@ pub(crate) fn escrow_mirror_name(efp: &str) -> String { format!("{MIRROR_PREFIX}escrow-{efp}.json") } +pub(crate) fn record_mirror_name(efp: &str) -> String { + format!("{MIRROR_PREFIX}record-{efp}.json") +} + /// `tag` is a keyed hash of the space (see `hierarchy::dek_mirror_tag`), so /// the bucket listing doesn't name spaces. pub(crate) fn dek_mirror_name(tag: &str, dek_id: u32) -> String { @@ -283,6 +287,21 @@ impl AtRestStore { .map(|_| ()) } + /// Drop a generation's chain: the newest generation has no successor, + /// so a chain there is stale (a restored or garbage row). + pub async fn clear_chain(&self, generation: i32) -> Result<(), DbErr> { + let _w = self.writer().await; + at_rest_key::Entity::update_many() + .col_expr( + at_rest_key::Column::Chain, + sea_orm::sea_query::Expr::value(Option::>::None), + ) + .filter(at_rest_key::Column::Generation.eq(generation)) + .exec(&self.db) + .await + .map(|_| ()) + } + /// Any escrow row at all (sealed material that must block provisioning). pub async fn has_escrow_rows(&self) -> Result { Ok(at_rest_escrow::Entity::find().count(&self.db).await? > 0) From 27f9430304264cc76c23593fbaed3e2d7fcc3899 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 11:48:29 +0000 Subject: [PATCH 07/15] fix(TC-887): name the chain walk resolve_chain (single DID-resolution entry guard) (#326) --- tinycloud-core/src/at_rest/keyring.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index 2071fa0d..a1759030 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -343,7 +343,7 @@ fn candidates(db: &[at_rest_key::Model], mirrored: Vec) -> C /// error. /// /// Returns the rows for 1..=top as they must be stored, and the keys. -fn resolve( +fn resolve_chain( cands: &Candidates, top: &at_rest_key::Model, top_ark: Ark, @@ -439,7 +439,7 @@ fn unlock_under_w( let Ok(ark) = open_ark(w, &c.row.sealed, &id, g, &c.row.w_label) else { continue; }; - match resolve(cands, &c.row, ark, Some(w)) { + match resolve_chain(cands, &c.row, ark, Some(w)) { Ok(found) => return Ok(Some(found)), // The database's own newest row: a broken store, fail loudly. Err(e) if c.from_db => return Err(e), @@ -460,7 +460,7 @@ fn unlock_under_w( Ok(None) } -/// Store what `resolve` chose: insert generations the database lacks, and +/// Store what `resolve_chain` chose: insert generations the database lacks, and /// update the seal or chain of those it has. Every row was authenticated. async fn persist_chosen( store: &AtRestStore, @@ -1200,7 +1200,7 @@ impl AtRestKeyring { ))?; // Under a new root the rows still carry the old W: every older // generation must open by its chain, or recovery fails loudly. - let (mut chosen, keys) = resolve(&cands, &top, ark, None)?; + let (mut chosen, keys) = resolve_chain(&cands, &top, ark, None)?; for row in chosen.iter_mut() { let g = row.generation as u32; let gen = &keys.gens[&g]; From 9b54bb77a985ff662bb1f4df29883c680ce20d8d Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 12:55:52 +0000 Subject: [PATCH 08/15] fix(TC-887): never store mirror seal bytes; non-secret gate reasons (#326) - A generation picked from the mirror for its chain keeps the database's own seal (same ARK id), or gets a fresh seal under W if the database lacks it; mirror seal bytes are never stored. Test fails before. - GateRefusal::Unavailable carries AtRestError::reason() (a stable, key-free code), never raw store or mirror error text; the detail is logged server-side. --- tinycloud-core/src/at_rest/keyring.rs | 32 +++++++++++++++------ tinycloud-core/src/at_rest/keyring/tests.rs | 32 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index a1759030..a5bd8dc0 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -666,14 +666,22 @@ impl AtRestKeyring { } let cands = candidates(&db, mirrored); if let Some((mut chosen, keys)) = unlock_under_w(&self.w, db.len() as u32, &cands)? { - // A generation the database lacks is stored with a fresh seal - // under W, never with the mirror's bytes. + // Only chains and ARK ids come from the chosen rows. Seal bytes + // are never taken from the mirror: a generation the database has + // keeps its own seal, and one it lacks gets a fresh seal under W. for row in chosen.iter_mut() { - if !db.iter().any(|d| d.generation == row.generation) { - let g = row.generation as u32; - let gen = &keys.gens[&g]; - row.sealed = seal_ark(&self.w, &gen.ark, &gen.ark_id, g); - row.w_label = W_LABEL.into(); + match db.iter().find(|d| d.generation == row.generation) { + Some(d) if d.ark_id == row.ark_id => { + row.sealed = d.sealed.clone(); + row.w_label = d.w_label.clone(); + } + Some(_) => {} // persist_chosen refuses this + None => { + let g = row.generation as u32; + let gen = &keys.gens[&g]; + row.sealed = seal_ark(&self.w, &gen.ark, &gen.ark_id, g); + row.w_label = W_LABEL.into(); + } } } persist_chosen(&self.store, &db, &chosen).await?; @@ -1070,7 +1078,10 @@ impl AtRestKeyring { "no active escrow bundle; restart the node to escrow the key".into(), )) } - Err(e) => return Err(GateRefusal::Unavailable(e.to_string())), + Err(e) => { + tracing::warn!(error = %e, "at-rest escrow gate could not read the escrow"); + return Err(GateRefusal::Unavailable(e.reason())); + } }; if bundle.holders() != self.config.holders { return Err(GateRefusal::RecipientsMismatch); @@ -1101,7 +1112,10 @@ impl AtRestKeyring { Ok(None) => Err(GateRefusal::NotVerified { efp: efp.to_string(), }), - Err(e) => Err(GateRefusal::Unavailable(e.to_string())), + Err(e) => { + tracing::warn!(error = %e, "at-rest escrow gate could not read the ceremony record"); + Err(GateRefusal::Unavailable(e.reason())) + } } } diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index ad9be5a3..a0d168ca 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -546,6 +546,38 @@ async fn a_restore_before_the_escrow_or_record_needs_no_new_ceremony() { assert!(!view.verified); } +#[tokio::test] +async fn a_mirrored_row_picked_for_its_chain_never_brings_its_seal() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + k.rotate_ark().await.unwrap(); + drop(k); + let gen1 = AtRestStore::new(db.clone()).keys().await.unwrap()[0].clone(); + // The database lost generation 1's chain; the mirror has it, next to + // junk seal bytes. + set_chain(&db, 1, None).await; + let name = ark_mirror_name(1, &hex::encode(&gen1.ark_id)); + let mut obj: serde_json::Value = + serde_json::from_slice(&mirror.get(&name).await.unwrap().unwrap()).unwrap(); + obj["sealed"] = serde_json::json!(base64::engine::general_purpose::STANDARD.encode([7; 60])); + mirror + .put(&name, serde_json::to_vec(&obj).unwrap()) + .await + .unwrap(); + + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + let rows = AtRestStore::new(db.clone()).keys().await.unwrap(); + assert_eq!( + rows[0].sealed, gen1.sealed, + "the database keeps its own seal" + ); + assert_eq!( + rows[0].chain, gen1.chain, + "the authenticated chain is restored" + ); +} + #[tokio::test] async fn a_stale_chain_on_the_newest_generation_is_cleared_not_a_permanent_conflict() { let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); From c296f665db996b222ebcf11d2148cdd8545a9ccb Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 13:10:26 +0000 Subject: [PATCH 09/15] fix(TC-887): mirror contents never stop a node with a healthy database (#326) - Mirrored ARK objects that can't be read are skipped with a warning whenever the database has rows (still fatal with an empty database, where the mirror is all there is). Record and PITR-bundle restores skip unreadable objects too. - KeyringConfig::require_mirror: boot and recovery mirror writes are fatal only when keys may seal data; otherwise they are logged. - MemoryMirror: plant_unreadable and set_read_only for tests. - Tests: odd names, a directory, a symlink and an unreadable record on a healthy database (pinned, unpinned, locked recovery); fatal with an empty database; a read-only mirror is fatal only with require_mirror. --- tinycloud-core/src/at_rest/keyring.rs | 66 +++++++++++++++------ tinycloud-core/src/at_rest/keyring/tests.rs | 60 +++++++++++++++++++ tinycloud-core/src/at_rest/store.rs | 21 +++++++ 3 files changed, 128 insertions(+), 19 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index a5bd8dc0..033b2e04 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -48,6 +48,11 @@ pub struct KeyringConfig { /// The TEE identity written into verification records (informational). pub app_id: Option, pub compose_hash: Option, + /// Whether mirror writes at boot and after recovery must succeed. True + /// once keys may seal data (mode other than `off`); with it false, a + /// mirror that can't be written is logged and the database alone serves, + /// so mirror contents never stop a node that encrypts nothing. + pub require_mirror: bool, } #[derive(Debug, thiserror::Error)] @@ -500,14 +505,26 @@ async fn persist_chosen( } /// Mirrored ARK rows, unauthenticated. Junk objects (not JSON, a newer -/// format, bad fields) are skipped with a warning; only mirror I/O fails. -async fn mirrored_rows(mirror: &dyn KeyMirror) -> Result, AtRestError> { +/// format, bad fields) are always skipped with a warning. An object that +/// can't be read at all is skipped too when `tolerate_unreadable` (the +/// database already has rows, so the mirror is only a supplement); +/// otherwise it fails the call, as does a failure to list. +async fn mirrored_rows( + mirror: &dyn KeyMirror, + tolerate_unreadable: bool, +) -> Result, AtRestError> { let mut names = mirror.list(ARK_MIRROR_PREFIX).await.map_err(mirror_err)?; names.sort(); let mut rows = Vec::new(); for name in names { - let Some(bytes) = get_capped(mirror, &name).await.map_err(mirror_err)? else { - continue; + let bytes = match get_capped(mirror, &name).await { + Ok(Some(bytes)) => bytes, + Ok(None) => continue, + Err(error) if tolerate_unreadable => { + tracing::warn!(object = ?name, %error, "ignoring an at-rest mirror object that can't be read"); + continue; + } + Err(error) => return Err(mirror_err(error)), }; let parsed = serde_json::from_slice::(&bytes) .map_err(|e| e.to_string()) @@ -622,7 +639,7 @@ impl AtRestKeyring { async fn load(&self) -> Result<(), AtRestError> { let mut db = self.store.keys().await?; check_contiguous(&db)?; - let mirrored = mirrored_rows(&*self.mirror).await?; + let mirrored = mirrored_rows(&*self.mirror, !db.is_empty()).await?; if db.is_empty() && mirrored.is_empty() { if self.store.has_space_keys().await? || self.store.has_escrow_rows().await? @@ -686,7 +703,7 @@ impl AtRestKeyring { } persist_chosen(&self.store, &db, &chosen).await?; for row in self.store.keys().await? { - Self::mirror_row(&*self.mirror, &row).await?; + self.mirror_soft(Self::mirror_row(&*self.mirror, &row).await)?; } *self.phase.write() = Phase::Unlocked(Arc::new(keys)); return Ok(()); @@ -735,6 +752,17 @@ impl AtRestKeyring { } } + /// A boot-time mirror write: fatal only when `require_mirror`. + fn mirror_soft(&self, result: Result<(), AtRestError>) -> Result<(), AtRestError> { + match result { + Err(AtRestError::Mirror(error)) if !self.config.require_mirror => { + tracing::warn!(%error, "at-rest key mirror could not be written; continuing (nothing is encrypted)"); + Ok(()) + } + other => other, + } + } + fn keys(&self) -> Result, AtRestError> { match &*self.phase.read() { Phase::Unlocked(keys) => Ok(keys.clone()), @@ -786,7 +814,7 @@ impl AtRestKeyring { if bundle.holders() != self.config.holders { return Err(AtRestError::EscrowHoldersChanged); } - self.mirror_escrow(&row).await?; + self.mirror_soft(self.mirror_escrow(&row).await)?; if row.record.is_none() { self.restore_record(&gen.ark, &bundle).await?; } @@ -827,7 +855,7 @@ impl AtRestKeyring { created_at: now_rfc3339(), }; self.store.activate_escrow(row.clone()).await?; - self.mirror_escrow(&row).await?; + self.mirror_soft(self.mirror_escrow(&row).await)?; if created { tracing::warn!( generation = g, @@ -857,11 +885,13 @@ impl AtRestKeyring { /// MAC verifies under the ARK. async fn restore_record(&self, ark: &Ark, bundle: &EscrowBundle) -> Result<(), AtRestError> { let efp = bundle.fingerprint().to_string(); - let Some(bytes) = get_capped(&*self.mirror, &record_mirror_name(&efp)) - .await - .map_err(mirror_err)? - else { - return Ok(()); + let bytes = match get_capped(&*self.mirror, &record_mirror_name(&efp)).await { + Ok(Some(bytes)) => bytes, + Ok(None) => return Ok(()), + Err(error) => { + tracing::warn!(%efp, %error, "can't read the mirrored ceremony record; not restoring it"); + return Ok(()); + } }; match VerificationRecord::from_json(&bytes) { Ok(record) if record.verify(ark, bundle).is_ok() => { @@ -886,16 +916,14 @@ impl AtRestKeyring { .await .map_err(mirror_err)?; for name in names { - let Some(bytes) = get_capped(&*self.mirror, &name).await.map_err(mirror_err)? else { + let Ok(Some(bytes)) = get_capped(&*self.mirror, &name).await else { continue; }; let Ok(record) = VerificationRecord::from_json(&bytes) else { continue; }; let efp = record.efp.to_string(); - let Some(bundle_bytes) = get_capped(&*self.mirror, &escrow_mirror_name(&efp)) - .await - .map_err(mirror_err)? + let Ok(Some(bundle_bytes)) = get_capped(&*self.mirror, &escrow_mirror_name(&efp)).await else { continue; }; @@ -1204,7 +1232,7 @@ impl AtRestKeyring { "recovered ARK is not the newest generation", )); } - let cands = candidates(&db, mirrored_rows(&*self.mirror).await?); + let cands = candidates(&db, mirrored_rows(&*self.mirror, !db.is_empty()).await?); let top = cands .get(&bundle.generation) .and_then(|l| l.iter().find(|c| c.row.ark_id == bundle.ark_id.0)) @@ -1223,7 +1251,7 @@ impl AtRestKeyring { } persist_chosen(&self.store, &db, &chosen).await?; for row in self.store.keys().await? { - Self::mirror_row(&*self.mirror, &row).await?; + self.mirror_soft(Self::mirror_row(&*self.mirror, &row).await)?; } *self.phase.write() = Phase::Unlocked(Arc::new(keys)); tracing::warn!( diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index a0d168ca..b681b89b 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -65,6 +65,7 @@ fn config(h: &Holders, pinned: Option) -> KeyringConfig { pinned_efp: pinned, app_id: Some("app".into()), compose_hash: Some("compose".into()), + require_mirror: true, } } @@ -1304,3 +1305,62 @@ async fn migration_down_refuses_to_drop_keys() { .unwrap(); assert!(at_rest_key::Entity::find().all(&db).await.is_err()); } + +#[tokio::test] +async fn unreadable_mirror_objects_never_stop_a_node_with_a_healthy_database() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let dek = k.current_key("space-a").await.unwrap(); + let efp = k.escrow_view().await.unwrap().unwrap().efp; + drop(k); + // Objects the real mirror can't read: odd names, a directory, a symlink. + for name in [ + "ark-0000000002-x y.json", + "ark-..json", + "ark-0000000002-x+y.json", + "ark-0000000002-d.json", + "record-x y.json", + ] { + mirror.plant_unreadable(&format!("{MIRROR_PREFIX}{name}")); + } + for pin in [None, Some(efp)] { + let k = open(&db, &mirror, &ROOT, config(&h, pin)).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + assert_eq!(k.current_key("space-a").await.unwrap().bytes(), dek.bytes()); + } + // A new root still boots locked and recovers. + let k = open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))) + .await + .unwrap(); + submit(&k, &h, 0).await.unwrap(); + assert_eq!(submit(&k, &h, 1).await.unwrap(), ShareOutcome::Unlocked); + drop(k); + + // With an empty database the mirror is all there is: an object that + // can't be read stops the boot rather than guessing. + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + assert!(matches!( + open(&db, &mirror, &NEW_ROOT, config(&h, Some(efp))).await, + Err(AtRestError::Mirror(_)) + )); +} + +#[tokio::test] +async fn a_mirror_that_cannot_be_written_only_matters_once_keys_may_seal() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + // The mirror lost its copies and can't take new ones. + mirror.clear(); + mirror.set_read_only(true); + let off = KeyringConfig { + require_mirror: false, + ..config(&h, None) + }; + let k = open(&db, &mirror, &ROOT, off).await.unwrap(); + assert_eq!(k.state(), KeyringState::Unlocked); + drop(k); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::Mirror(_)) + )); +} diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs index 60fef976..20d2dab4 100644 --- a/tinycloud-core/src/at_rest/store.rs +++ b/tinycloud-core/src/at_rest/store.rs @@ -434,6 +434,8 @@ impl AtRestStore { pub struct MemoryMirror { objects: parking_lot::Mutex>>, fail: std::sync::atomic::AtomicBool, + read_only: std::sync::atomic::AtomicBool, + unreadable: parking_lot::Mutex>, } #[cfg(any(test, feature = "test-util"))] @@ -443,6 +445,19 @@ impl MemoryMirror { self.fail.store(fail, std::sync::atomic::Ordering::SeqCst); } + /// Refuse every write, like a bucket whose objects can't be replaced. + pub fn set_read_only(&self, read_only: bool) { + self.read_only + .store(read_only, std::sync::atomic::Ordering::SeqCst); + } + + /// List `name` but fail to read it, like an object whose name or type + /// the real mirror refuses. + pub fn plant_unreadable(&self, name: &str) { + self.objects.lock().insert(name.to_owned(), Vec::new()); + self.unreadable.lock().insert(name.to_owned()); + } + pub fn names(&self) -> Vec { self.objects.lock().keys().cloned().collect() } @@ -470,12 +485,18 @@ impl MemoryMirror { impl KeyMirror for MemoryMirror { async fn put(&self, name: &str, bytes: Vec) -> Result<(), MirrorError> { self.check()?; + if self.read_only.load(std::sync::atomic::Ordering::SeqCst) { + return Err("mirror is read-only".into()); + } self.objects.lock().insert(name.to_owned(), bytes); Ok(()) } async fn get(&self, name: &str) -> Result>, MirrorError> { self.check()?; + if self.unreadable.lock().contains(name) { + return Err(format!("cannot read {name:?}").into()); + } match self.objects.lock().get(name) { Some(bytes) if bytes.len() > MAX_MIRROR_OBJECT => Err(Box::new(MirrorObjectTooLarge)), other => Ok(other.cloned()), From 17e1091a69b3714c355f75c92a13a120278fd4f6 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 13:19:13 +0000 Subject: [PATCH 10/15] chore(TC-887): rename the at-rest migration to m20261017_010000_at_rest_keys (#326) Orders after #341's m20261017_000000_epoch_order_space_index. No schema change. --- ...00000_at_rest_keys.rs => m20261017_010000_at_rest_keys.rs} | 0 tinycloud-core/src/migrations/mod.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename tinycloud-core/src/migrations/{m20261017_000000_at_rest_keys.rs => m20261017_010000_at_rest_keys.rs} (100%) diff --git a/tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs b/tinycloud-core/src/migrations/m20261017_010000_at_rest_keys.rs similarity index 100% rename from tinycloud-core/src/migrations/m20261017_000000_at_rest_keys.rs rename to tinycloud-core/src/migrations/m20261017_010000_at_rest_keys.rs diff --git a/tinycloud-core/src/migrations/mod.rs b/tinycloud-core/src/migrations/mod.rs index 6adf6c9a..345e7833 100644 --- a/tinycloud-core/src/migrations/mod.rs +++ b/tinycloud-core/src/migrations/mod.rs @@ -26,7 +26,7 @@ pub mod m20261007_000000_database_alias; pub mod m20261007_010000_database_identity_fence; pub mod m20261014_000000_delegation_cid_alias; pub mod m20261015_000000_delegation_cursor_index; -pub mod m20261017_000000_at_rest_keys; +pub mod m20261017_010000_at_rest_keys; pub struct Migrator; @@ -61,7 +61,7 @@ impl MigratorTrait for Migrator { Box::new(m20261007_010000_database_identity_fence::Migration), Box::new(m20261014_000000_delegation_cid_alias::Migration), Box::new(m20261015_000000_delegation_cursor_index::Migration), - Box::new(m20261017_000000_at_rest_keys::Migration), + Box::new(m20261017_010000_at_rest_keys::Migration), ] } } From 9f17af351f2f5907ba9a08c1efeed1eb576d67fc Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 14:03:20 +0000 Subject: [PATCH 11/15] fix(TC-887): never replace a mirrored DEK; guarded pre-ceremony re-split (#326) - D1: a mirrored data key that doesn't authenticate is never ignored and never overwritten. It may be the genuine key under a generation that didn't load (a restore plus a failed boot read). restore_dek fails closed for that space (AtRestError::DekUnavailable, transient, reason dek_unavailable) instead of letting create_dek and mirror_dek replace it; mirror_dek never overwrites an existing object with other bytes; an oversized DEK object fails closed too. Test reproduces the review's D1 chain and checks the original key returns once the object reads. - KeyringConfig::allow_resplit_before_verify: a holder change may re-split at boot only with the flag, no pin, an unverified active escrow and no data key ever made; otherwise EscrowHoldersChanged. - Merges main (#341's m20261017_000000_epoch_order_space_index and its migrated_through m20261015 test). --- tinycloud-core/src/at_rest/keyring.rs | 114 +++++++++++++------ tinycloud-core/src/at_rest/keyring/tests.rs | 117 ++++++++++++++++++-- tinycloud-core/src/at_rest/store.rs | 10 ++ 3 files changed, 198 insertions(+), 43 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index 033b2e04..59896425 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -53,6 +53,12 @@ pub struct KeyringConfig { /// mirror that can't be written is logged and the database alone serves, /// so mirror contents never stop a node that encrypts nothing. pub require_mirror: bool, + /// `at_rest.escrow.allow_resplit_before_verify`: let a holder change + /// re-split at boot, but only while nothing is pinned, the active + /// escrow has no verified ceremony record, and no data key has ever + /// been made (so nothing was ever sealed). For fixing a wrong holder + /// before the ceremony; otherwise a holder change refuses the boot. + pub allow_resplit_before_verify: bool, } #[derive(Debug, thiserror::Error)] @@ -67,8 +73,10 @@ pub enum AtRestError { Conflict(&'static str), #[error("at_rest.escrow.verified_fp pins an escrow that is not for the configured recipients and the newest key; the node never re-splits while a fingerprint is pinned")] EscrowPinned, - #[error("the configured escrow recipients differ from the active escrow bundle's; the node never re-splits on its own (an explicit re-escrow is needed)")] + #[error("the configured escrow recipients differ from the active escrow bundle's; the node never re-splits on its own (before the ceremony, at_rest.escrow.allow_resplit_before_verify allows it; afterwards an explicit re-escrow is needed)")] EscrowHoldersChanged, + #[error("a space's data key is unavailable: {0}")] + DekUnavailable(&'static str), #[error("at-rest escrow: {0}")] Escrow(#[from] EscrowError), #[error("at-rest keys are locked")] @@ -89,6 +97,7 @@ impl AtRestError { | Self::Conflict(_) | Self::Locked | Self::NotProvisioned + | Self::DekUnavailable(_) ) } @@ -102,6 +111,7 @@ impl AtRestError { Self::Conflict(why) => format!("conflict: {why}"), Self::EscrowPinned => "escrow_pinned".into(), Self::EscrowHoldersChanged => "escrow_holders_changed".into(), + Self::DekUnavailable(why) => format!("dek_unavailable: {why}"), Self::Escrow(e) => format!("escrow: {e}"), Self::Locked => "locked".into(), Self::NotProvisioned => "not_provisioned".into(), @@ -811,14 +821,20 @@ impl AtRestKeyring { "the active escrow bundle is for another ARK", )); } - if bundle.holders() != self.config.holders { - return Err(AtRestError::EscrowHoldersChanged); + if bundle.holders() == self.config.holders { + self.mirror_soft(self.mirror_escrow(&row).await)?; + if row.record.is_none() { + self.restore_record(&gen.ark, &bundle).await?; + } + return Ok(()); } - self.mirror_soft(self.mirror_escrow(&row).await)?; - if row.record.is_none() { - self.restore_record(&gen.ark, &bundle).await?; + if !self.may_resplit(&bundle).await? { + return Err(AtRestError::EscrowHoldersChanged); } - return Ok(()); + tracing::warn!( + generation = g, + "re-splitting an unverified escrow for corrected holders (allow_resplit_before_verify); the ceremony starts over" + ); } let wanted = |b: &EscrowBundle| { b.holders() == self.config.holders && b.ark_id == gen.ark_id && b.generation == g @@ -881,6 +897,16 @@ impl AtRestKeyring { Ok(()) } + /// A holder change may re-split only when explicitly allowed, nothing is + /// pinned, the active escrow was never verified, and no data key has + /// ever been made. + async fn may_resplit(&self, active: &EscrowBundle) -> Result { + Ok(self.config.allow_resplit_before_verify + && self.config.pinned_efp.is_none() + && self.verified_record(active).await?.is_none() + && !self.store.has_space_keys().await?) + } + /// A ceremony record for `bundle` from the mirror, stored only if its /// MAC verifies under the ARK. async fn restore_record(&self, ark: &Ark, bundle: &EscrowBundle) -> Result<(), AtRestError> { @@ -1348,8 +1374,29 @@ impl AtRestKeyring { Some((row, key)) } + /// The mirror object holding a space's DEK, as bytes. An object that is + /// present but too large to be one fails closed for the space: it must + /// never read as "absent", or a new DEK would be made and mirrored over + /// it. + async fn dek_object(&self, name: &str) -> Result>, DataKeyError> { + match self.mirror.get(name).await { + Ok(bytes) => Ok(bytes), + Err(e) if e.is::() => Err(backend(AtRestError::DekUnavailable( + "the mirrored object for this data key is oversized", + ))), + Err(e) => Err(backend(mirror_err(e))), + } + } + /// A space's DEK row from the mirror, put back into the database once - /// it authenticates. Anything else in the mirror is ignored (warned). + /// it authenticates. + /// + /// A mirrored object that doesn't authenticate is **never ignored**: it + /// may be the genuine key under an ARK generation this node hasn't + /// loaded (a restore plus a failed boot read), and making a new DEK + /// would strand everything sealed under it. The space fails closed + /// ([`AtRestError::DekUnavailable`], a 503) until the generation loads + /// or an operator removes the object. async fn restore_dek( &self, keys: &Keys, @@ -1357,18 +1404,25 @@ impl AtRestKeyring { dek_id: u32, ) -> Result, DataKeyError> { let name = dek_mirror_name(&keys.dek_tag.tag(space), dek_id); - let Some(bytes) = get_capped(&*self.mirror, &name) - .await - .map_err(|e| backend(mirror_err(e)))? - else { + let Some(bytes) = self.dek_object(&name).await? else { return Ok(None); }; let Some((row, _)) = Self::authentic_mirrored_dek(keys, space, dek_id, &bytes) else { - tracing::warn!( + let newest = keys.newest().0; + let names_unloaded_generation = + serde_json::from_slice::(&bytes).is_ok_and(|m| m.generation > newest); + tracing::error!( dek_id, - "ignoring a mirrored data key that does not authenticate for its space" + names_unloaded_generation, + "a mirrored data key for this space does not authenticate; refusing to make a new one" ); - return Ok(None); + return Err(backend(AtRestError::DekUnavailable( + if names_unloaded_generation { + "the mirrored data key names an ARK generation this node hasn't loaded; restart once the key mirror is readable" + } else { + "the mirror holds a data key object for this space that does not authenticate" + }, + ))); }; tracing::warn!( dek_id, @@ -1382,10 +1436,9 @@ impl AtRestKeyring { } /// Make sure the mirror holds exactly this DEK row before any object is - /// sealed under it. A *genuine* different DEK under the same id means - /// objects may exist under another key: refuse, never overwrite. A - /// mirror object that doesn't authenticate carries no key and is - /// replaced. + /// sealed under it. An existing object is never overwritten: if it + /// isn't this exact row (a genuine different key, junk, or a key under + /// a generation not loaded), the space fails closed. async fn mirror_dek( &self, keys: &Keys, @@ -1393,24 +1446,15 @@ impl AtRestKeyring { ) -> Result<(), DataKeyError> { let dek_id = row.dek_id as u32; let name = dek_mirror_name(&keys.dek_tag.tag(&row.space), dek_id); - let existing = get_capped(&*self.mirror, &name) - .await - .map_err(|e| backend(mirror_err(e)))?; - if let Some(bytes) = existing { - match Self::authentic_mirrored_dek(keys, &row.space, dek_id, &bytes) { + if let Some(bytes) = self.dek_object(&name).await? { + return match Self::authentic_mirrored_dek(keys, &row.space, dek_id, &bytes) { Some((m, _)) if m.generation == row.generation && m.wrapped == row.wrapped => { - return Ok(()) - } - Some(_) => { - return Err(backend(AtRestError::Inconsistent( - "the mirror holds a different data key with this id", - ))) + Ok(()) } - None => tracing::warn!( - dek_id, - "replacing a mirrored data key that does not authenticate for its space" - ), - } + _ => Err(backend(AtRestError::DekUnavailable( + "the mirror holds different bytes for this data key; it is never overwritten", + ))), + }; } let body = serde_json::to_vec(&MirroredDek::from_row(row)).expect("serializable"); self.mirror diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index b681b89b..74792c8c 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -66,6 +66,7 @@ fn config(h: &Holders, pinned: Option) -> KeyringConfig { app_id: Some("app".into()), compose_hash: Some("compose".into()), require_mirror: true, + allow_resplit_before_verify: false, } } @@ -1225,19 +1226,119 @@ async fn a_mirrored_data_key_for_another_space_is_never_stored() { let copied = mirror.get(&a_name).await.unwrap().unwrap(); mirror.put(&b_name, copied.clone()).await.unwrap(); - // It doesn't authenticate for space-b: not stored, not served, replaced. - assert!(matches!( - k.key("space-b", DekId::new(1)).await, - Err(DataKeyError::UnknownKey { .. }) - )); + // It doesn't authenticate for space-b: never stored, never replaced, + // and space-b fails closed rather than getting a new key over it. + let unavailable = |r: Result| { + matches!(r, Err(DataKeyError::Backend(e)) + if matches!(e.downcast_ref::(), Some(AtRestError::DekUnavailable(_)))) + }; + assert!(unavailable(k.key("space-b", DekId::new(1)).await)); + assert!(unavailable(k.current_key("space-b").await)); assert!(AtRestStore::new(db.clone()) .dek("space-b", 1) .await .unwrap() .is_none()); - let b = k.current_key("space-b").await.unwrap(); - assert_ne!(b.bytes(), a.bytes()); - assert_ne!(mirror.get(&b_name).await.unwrap().unwrap(), copied); + assert_eq!(mirror.get(&b_name).await.unwrap().unwrap(), copied); + assert_eq!(k.current_key("space-a").await.unwrap().bytes(), a.bytes()); +} + +/// The round-3 correctness probe D1: a restore before generation 2, plus +/// one failed read of the genuine generation-2 object at boot, used to +/// replace space-x's data key and overwrite its only copy. +#[tokio::test] +async fn a_mirrored_data_key_under_an_unloaded_generation_is_never_replaced() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + k.rotate_ark().await.unwrap(); + let original = k.current_key("space-x").await.unwrap(); + let keys = k.keys().unwrap(); + let dek_name = dek_mirror_name(&keys.dek_tag.tag("space-x"), 1); + let gen2 = AtRestStore::new(db.clone()).keys().await.unwrap()[1].clone(); + let gen2_name = ark_mirror_name(2, &hex::encode(&gen2.ark_id)); + drop((k, keys)); + let mirrored = mirror.get(&dek_name).await.unwrap().unwrap(); + + // The restore: generation 2, its escrow and space-x's DEK row are gone. + at_rest_key::Entity::delete_by_id(2) + .exec(&db) + .await + .unwrap(); + at_rest_escrow::Entity::delete_many() + .filter(at_rest_escrow::Column::Generation.eq(2)) + .exec(&db) + .await + .unwrap(); + space_data_key::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + // One boot where the genuine generation-2 object can't be read. + mirror.set_unreadable(&gen2_name, true); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.keys().unwrap().newest().0, 1); + for _ in 0..2 { + assert!(matches!( + k.current_key("space-x").await, + Err(DataKeyError::Backend(e)) + if matches!(e.downcast_ref::(), Some(AtRestError::DekUnavailable(why)) if why.contains("hasn't loaded")) + )); + } + assert_eq!(mirror.get(&dek_name).await.unwrap().unwrap(), mirrored); + assert!(space_data_key::Entity::find() + .all(&db) + .await + .unwrap() + .is_empty()); + drop(k); + + // The object is readable again: generation 2 and the original key are back. + mirror.set_unreadable(&gen2_name, false); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!(k.keys().unwrap().newest().0, 2); + assert_eq!( + k.current_key("space-x").await.unwrap().bytes(), + original.bytes() + ); +} + +#[tokio::test] +async fn a_wrong_holder_can_be_fixed_only_before_the_ceremony_and_any_sealing() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let wrong = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let first = wrong.escrow_view().await.unwrap().unwrap().efp; + drop(wrong); + let fixed = holders(); + let allow = |h: &Holders| KeyringConfig { + allow_resplit_before_verify: true, + ..config(h, None) + }; + // Without the flag a holder change is refused, as before. + assert!(matches!( + open(&db, &mirror, &ROOT, config(&fixed, None)).await, + Err(AtRestError::EscrowHoldersChanged) + )); + // With it, unverified and nothing sealed: re-split for the new holders. + let k = open(&db, &mirror, &ROOT, allow(&fixed)).await.unwrap(); + let second = k.escrow_view().await.unwrap().unwrap().efp; + assert_ne!(second, first); + // Once verified, the flag no longer helps. + verify_escrow(&k, &fixed).await; + drop(k); + assert!(matches!( + open(&db, &mirror, &ROOT, allow(&h)).await, + Err(AtRestError::EscrowHoldersChanged) + )); + + // Unverified, but a data key exists (data may be sealed): refused. + let (db, mirror) = (self::db().await, Arc::new(MemoryMirror::default())); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + k.current_key("space-a").await.unwrap(); + drop(k); + assert!(matches!( + open(&db, &mirror, &ROOT, allow(&fixed)).await, + Err(AtRestError::EscrowHoldersChanged) + )); } #[tokio::test] diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs index 20d2dab4..8a945ca5 100644 --- a/tinycloud-core/src/at_rest/store.rs +++ b/tinycloud-core/src/at_rest/store.rs @@ -458,6 +458,16 @@ impl MemoryMirror { self.unreadable.lock().insert(name.to_owned()); } + /// Make reads of an existing object fail (a transient error on a + /// genuine object), or work again with `false`. + pub fn set_unreadable(&self, name: &str, unreadable: bool) { + if unreadable { + self.unreadable.lock().insert(name.to_owned()); + } else { + self.unreadable.lock().remove(name); + } + } + pub fn names(&self) -> Vec { self.objects.lock().keys().cloned().collect() } From eee68da3d691a99f8f60419a01885b77f98a72be Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 15:58:40 +0000 Subject: [PATCH 12/15] fix(TC-887): pre-ceremony holder fix replaces the ARK; take S3's write-lock guards (#326) - allow_resplit_before_verify no longer re-splits the same ARK: the node makes a fresh ARK (generation 1), splits it for the configured holders and retires the old one, so old shares rebuild only a key that never protected anything. - A W-authenticated tombstone per old ARK goes to the mirror first; load() then drops that ARK's key and escrow rows (also from a database restored to before the retirement) and ignores and deletes its ARK, escrow and record objects. Forged or junk tombstones are skipped; a database mixing retired and live rows is Inconsistent. - The database swaps keys in one transaction (replace_all_keys). - may_resplit also reads the mirror: any record- or dek- object, or any stored record, refuses the shortcut. - KeyMirror::delete (MemoryMirror implements it). - Takes S3's #333 keyring changes (efa5bbb): write_locked / in_write_lock, DataKeyError::NotPrepared, the guards on restore_dek, mirror_dek and create_dek, DataKeyProvider::prepare_read and its test. D1 still holds: prepare_read's restore fails closed on a mirrored DEK that doesn't authenticate. --- tinycloud-core/src/at_rest/hierarchy.rs | 23 ++ tinycloud-core/src/at_rest/keyring.rs | 245 ++++++++++++++++++-- tinycloud-core/src/at_rest/keyring/tests.rs | 166 ++++++++++++- tinycloud-core/src/at_rest/mod.rs | 2 +- tinycloud-core/src/at_rest/provider.rs | 30 +++ tinycloud-core/src/at_rest/store.rs | 89 +++++++ tinycloud-core/src/at_rest/testing.rs | 4 + 7 files changed, 530 insertions(+), 29 deletions(-) diff --git a/tinycloud-core/src/at_rest/hierarchy.rs b/tinycloud-core/src/at_rest/hierarchy.rs index 4854d71a..18ec8ba8 100644 --- a/tinycloud-core/src/at_rest/hierarchy.rs +++ b/tinycloud-core/src/at_rest/hierarchy.rs @@ -37,6 +37,7 @@ const CHAIN_INFO: &[u8] = b"tinycloud/at-rest/ark-chain/v1"; const SEAL_DOMAIN: &[u8] = b"tinycloud/at-rest/ark-seal/v1"; const CHAIN_DOMAIN: &[u8] = b"tinycloud/at-rest/ark-chain-aad/v1"; const DEK_DOMAIN: &[u8] = b"tinycloud/at-rest/dek/v1"; +const RETIRED_INFO: &[u8] = b"tinycloud/at-rest/retired/v1"; const NONCE_LEN: usize = 12; /// An AEAD open failed: wrong key, wrong binding, or tampered bytes. @@ -101,6 +102,28 @@ impl WrapKey { pub fn derive(root: &[u8]) -> Self { Self(hkdf32(root, W_LABEL.as_bytes())) } + + fn retire_mac(&self, ark_id: &[u8]) -> hmac::Hmac { + use hmac::Mac as _; + let key = hkdf32(self.0.as_ref(), RETIRED_INFO); + let mut mac = as hmac::Mac>::new_from_slice(key.as_ref()) + .expect("HMAC takes any key length"); + mac.update(ark_id); + mac + } + + /// A tag only this root can make: "this ARK was retired". A retired + /// ARK is never loaded again, even from a database restored to before + /// its retirement. + pub fn retire_tag(&self, ark_id: &[u8]) -> Vec { + use hmac::Mac as _; + self.retire_mac(ark_id).finalize().into_bytes().to_vec() + } + + pub fn verify_retire_tag(&self, ark_id: &[u8], tag: &[u8]) -> bool { + use hmac::Mac as _; + self.retire_mac(ark_id).verify_slice(tag).is_ok() + } } fn seal_aad(ark_id: &ArkId, generation: u32, w_label: &str) -> Vec { diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index 59896425..9595a6bb 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -22,10 +22,11 @@ use super::hierarchy::{ Kek, WrapKey, W_LABEL, }; use super::store::{ - ark_mirror_name, dek_mirror_name, escrow_mirror_name, record_mirror_name, AtRestStore, - KeyMirror, MirrorObjectTooLarge, MirroredArk, MirroredDek, ARK_MIRROR_PREFIX, MIRROR_PREFIX, + ark_mirror_name, dek_mirror_name, escrow_mirror_name, record_mirror_name, retired_mirror_name, + AtRestStore, KeyMirror, MirrorObjectTooLarge, MirroredArk, MirroredDek, MirroredRetired, + ARK_MIRROR_PREFIX, MIRROR_PREFIX, RETIRED_MIRROR_PREFIX, }; -use super::{DataKey, DataKeyError, DataKeyProvider, DekId, ObjectNameKey}; +use super::{in_write_lock, DataKey, DataKeyError, DataKeyProvider, DekId, ObjectNameKey}; use crate::models::{at_rest_escrow, at_rest_key, space_data_key}; const DEK_CACHE: usize = 4096; @@ -54,10 +55,13 @@ pub struct KeyringConfig { /// so mirror contents never stop a node that encrypts nothing. pub require_mirror: bool, /// `at_rest.escrow.allow_resplit_before_verify`: let a holder change - /// re-split at boot, but only while nothing is pinned, the active - /// escrow has no verified ceremony record, and no data key has ever - /// been made (so nothing was ever sealed). For fixing a wrong holder - /// before the ceremony; otherwise a holder change refuses the boot. + /// replace the root key at boot, but only while nothing is pinned, no + /// ceremony record exists, and no data key has ever been made (database + /// and mirror both checked). The node makes a fresh ARK, splits it for + /// the configured holders and retires the old ARK and its escrow, so + /// old shares rebuild only a key that never protected anything. For + /// fixing a wrong holder before the ceremony; otherwise a holder change + /// refuses the boot. pub allow_resplit_before_verify: bool, } @@ -647,18 +651,45 @@ impl AtRestKeyring { /// Read the stored and mirrored rows and set the phase. async fn load(&self) -> Result<(), AtRestError> { + // A retired ARK (W-authenticated tombstone in the mirror) is never + // loaded, even from a database restored to before its retirement. + let retired = self.retired_ark_ids().await?; let mut db = self.store.keys().await?; + if db.iter().any(|r| retired.contains(&r.ark_id)) { + // Retirement replaces every generation at once, so a database + // holding both kinds was not written by this node. + if !db.iter().all(|r| retired.contains(&r.ark_id)) + || self.store.has_space_keys().await? + { + return Err(AtRestError::Inconsistent( + "a retired at-rest root key is stored next to live key material", + )); + } + tracing::warn!("dropping key and escrow rows of a retired at-rest root key"); + self.store.drop_retired_keys(&retired).await?; + db = self.store.keys().await?; + } + if !retired.is_empty() { + self.sweep_retired(&retired).await?; + } check_contiguous(&db)?; - let mirrored = mirrored_rows(&*self.mirror, !db.is_empty()).await?; + let mut mirrored = mirrored_rows(&*self.mirror, !db.is_empty()).await?; + mirrored.retain(|r| !retired.contains(&r.ark_id)); if db.is_empty() && mirrored.is_empty() { + let live_mirror_objects = self + .mirror + .list(MIRROR_PREFIX) + .await + .map_err(mirror_err)? + .into_iter() + .filter(|name| { + !name.starts_with(RETIRED_MIRROR_PREFIX) + && !retired.iter().any(|id| name.contains(&hex::encode(id))) + }) + .count(); if self.store.has_space_keys().await? || self.store.has_escrow_rows().await? - || !self - .mirror - .list(MIRROR_PREFIX) - .await - .map_err(mirror_err)? - .is_empty() + || live_mirror_objects > 0 { // Another process may have provisioned since we read: its // ARK row is written before anything else. @@ -833,8 +864,11 @@ impl AtRestKeyring { } tracing::warn!( generation = g, - "re-splitting an unverified escrow for corrected holders (allow_resplit_before_verify); the ceremony starts over" + "replacing an unverified root key for corrected holders (allow_resplit_before_verify): the old shares are worthless and the ceremony starts over" ); + self.retire_and_replace().await?; + // The fresh key has no escrow yet: this makes its bundle. + return Box::pin(self.establish_escrow()).await; } let wanted = |b: &EscrowBundle| { b.holders() == self.config.holders && b.ark_id == gen.ark_id && b.generation == g @@ -898,13 +932,140 @@ impl AtRestKeyring { } /// A holder change may re-split only when explicitly allowed, nothing is - /// pinned, the active escrow was never verified, and no data key has - /// ever been made. + /// pinned, no ceremony record exists, and no data key was ever made. + /// The database and the mirror are both checked (a database restored to + /// an earlier point may have lost a data key or record the mirror still + /// has). Any record or data-key object in the mirror counts, readable + /// or not: it refuses only this pre-ceremony shortcut. async fn may_resplit(&self, active: &EscrowBundle) -> Result { - Ok(self.config.allow_resplit_before_verify - && self.config.pinned_efp.is_none() - && self.verified_record(active).await?.is_none() - && !self.store.has_space_keys().await?) + if !self.config.allow_resplit_before_verify || self.config.pinned_efp.is_some() { + return Ok(false); + } + if self.verified_record(active).await?.is_some() + || self.store.has_space_keys().await? + || self.store.has_escrow_record().await? + { + return Ok(false); + } + for prefix in ["record-", "dek-"] { + let names = self + .mirror + .list(&format!("{MIRROR_PREFIX}{prefix}")) + .await + .map_err(mirror_err)?; + if !names.is_empty() { + return Ok(false); + } + } + Ok(true) + } + + /// Replace every ARK generation with a fresh ARK (generation 1) for a + /// pre-ceremony re-split, so a removed holder's old share, with any + /// other old share, rebuilds only a key that never protected anything + /// and is never loaded again. + /// + /// 1. A W-authenticated tombstone per old ARK goes to the mirror first: + /// from then on [`Self::load`] drops that ARK's rows (even from a + /// database restored to an earlier point) and ignores its objects. + /// 2. The database swaps to the fresh ARK in one transaction, dropping + /// every escrow row. + /// 3. Reloading deletes the old ARK, escrow and record objects from + /// the mirror and mirrors the fresh ARK. + /// + /// A crash between 1 and 2 leaves only retired rows, which the next + /// boot drops before provisioning a fresh ARK. + async fn retire_and_replace(&self) -> Result<(), AtRestError> { + for row in self.store.keys().await? { + let id = hex::encode(&row.ark_id); + let tag = self.w.retire_tag(&row.ark_id); + let body = + serde_json::to_vec(&MirroredRetired::new(id.clone(), &tag)).expect("serializable"); + self.mirror + .put(&retired_mirror_name(&id), body) + .await + .map_err(mirror_err)?; + } + let ark = Ark::generate(); + let ark_id = ArkId::generate(); + self.store + .replace_all_keys(at_rest_key::Model { + generation: 1, + ark_id: ark_id.0.to_vec(), + w_label: W_LABEL.into(), + sealed: seal_ark(&self.w, &ark, &ark_id, 1), + chain: None, + created_at: now_rfc3339(), + }) + .await?; + tracing::warn!(generation = 1, %ark_id, "at-rest root key replaced; the old one is retired"); + self.load().await + } + + /// ARK ids with a tombstone that authenticates under this node's W. + /// Junk or forged tombstones are skipped; a read error fails the call + /// (skipping could bring a retired ARK back). + async fn retired_ark_ids(&self) -> Result>, AtRestError> { + let mut ids = Vec::new(); + let names = self + .mirror + .list(RETIRED_MIRROR_PREFIX) + .await + .map_err(mirror_err)?; + for name in names { + let Some(bytes) = get_capped(&*self.mirror, &name).await.map_err(mirror_err)? else { + continue; + }; + let parts = serde_json::from_slice::(&bytes) + .ok() + .and_then(|m| m.parts()); + match parts { + Some((id, tag)) if self.w.verify_retire_tag(&id, &tag) => ids.push(id), + _ => { + tracing::warn!(object = ?name, "ignoring an at-rest retirement marker that doesn't authenticate") + } + } + } + Ok(ids) + } + + /// Delete the mirror objects of retired ARKs: their ARK copies, escrow + /// bundles and ceremony records. Idempotent; a failed delete leaves an + /// object [`Self::load`] already ignores. + async fn sweep_retired(&self, retired: &[Vec]) -> Result<(), AtRestError> { + let hexes: Vec = retired.iter().map(hex::encode).collect(); + let mut doomed = Vec::new(); + for name in self + .mirror + .list(ARK_MIRROR_PREFIX) + .await + .map_err(mirror_err)? + { + if hexes.iter().any(|h| name.ends_with(&format!("-{h}.json"))) { + doomed.push(name); + } + } + for name in self + .mirror + .list(&format!("{MIRROR_PREFIX}escrow-")) + .await + .map_err(mirror_err)? + { + let Ok(Some(bytes)) = get_capped(&*self.mirror, &name).await else { + continue; + }; + let Ok(bundle) = EscrowBundle::from_json(&bytes) else { + continue; + }; + if retired.iter().any(|id| id.as_slice() == bundle.ark_id.0) { + doomed.push(record_mirror_name(&bundle.fingerprint().to_string())); + doomed.push(name); + } + } + for name in doomed { + self.mirror_soft(self.mirror.delete(&name).await.map_err(mirror_err))?; + } + Ok(()) } /// A ceremony record for `bundle` from the mirror, stored only if its @@ -1403,6 +1564,9 @@ impl AtRestKeyring { space: &str, dek_id: u32, ) -> Result, DataKeyError> { + if in_write_lock() { + return Err(DataKeyError::NotPrepared); + } let name = dek_mirror_name(&keys.dek_tag.tag(space), dek_id); let Some(bytes) = self.dek_object(&name).await? else { return Ok(None); @@ -1444,6 +1608,9 @@ impl AtRestKeyring { keys: &Keys, row: &space_data_key::Model, ) -> Result<(), DataKeyError> { + if in_write_lock() { + return Err(DataKeyError::NotPrepared); + } let dek_id = row.dek_id as u32; let name = dek_mirror_name(&keys.dek_tag.tag(&row.space), dek_id); if let Some(bytes) = self.dek_object(&name).await? { @@ -1470,6 +1637,9 @@ impl AtRestKeyring { keys: &Keys, space: &str, ) -> Result { + if in_write_lock() { + return Err(DataKeyError::NotPrepared); + } let (g, gen) = keys.newest(); let (wrapped, _) = new_dek(&gen.kek, space, DekId::new(1), g, &gen.ark_id); self.store @@ -1571,6 +1741,39 @@ impl DataKeyProvider for AtRestKeyring { } } + async fn prepare_read(&self, space: &str) -> Result<(), DataKeyError> { + let keys = match self.keys() { + Ok(keys) => keys, + // Locked: sealed reads answer 503 themselves; legacy reads go on. + Err(AtRestError::Locked | AtRestError::NotProvisioned) => return Ok(()), + Err(e) => return Err(backend(e)), + }; + if self.current.lock().get(space).is_some() + || self + .store + .newest_dek(space) + .await + .map_err(backend)? + .is_some() + { + return Ok(()); + } + let _create = self.dek_create.lock().await; + if self + .store + .newest_dek(space) + .await + .map_err(backend)? + .is_none() + { + // A database restored to before the space's first DEK: objects + // may be sealed under the mirrored one. Spaces have exactly DEK 1 + // (see `create_dek`). + self.restore_dek(&keys, space, 1).await?; + } + Ok(()) + } + fn object_name_key(&self) -> Result { Ok(self.keys().map_err(locked)?.nk.clone()) } diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index 74792c8c..ee54c9c6 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use base64::Engine as _; use sea_orm::{ - ColumnTrait, ConnectOptions, Database, DatabaseConnection, EntityTrait, QueryFilter, + ColumnTrait, ConnectOptions, Database, DatabaseConnection, EntityTrait, IntoActiveModel, + QueryFilter, }; use sea_orm_migration::MigratorTrait; use tinycloud_at_rest_escrow::{wrap_share, EscrowBundle, HolderRecipient, SharePayload}; @@ -1043,6 +1044,53 @@ async fn data_keys_survive_a_restore_to_before_they_were_created() { assert_eq!(k.current_key("space-a").await.unwrap().bytes(), a.bytes()); } +/// TC-887 S3: inside `write_locked` (the invoke write lock and transaction) +/// the keyring never writes: a key that would need restoring or creating is +/// `NotPrepared`, with no row inserted. `prepare_read` restores it beforehand +/// (never creating one), after which lookups under the lock succeed. +#[tokio::test] +async fn under_the_write_lock_keys_are_only_looked_up() { + use crate::at_rest::write_locked; + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let a = k.current_key("space-a").await.unwrap(); + drop(k); + space_data_key::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + let rows = || async { space_data_key::Entity::find().all(&db).await.unwrap().len() }; + + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert!(matches!( + write_locked(k.key("space-a", DekId::new(1))).await, + Err(DataKeyError::NotPrepared) + )); + assert!(matches!( + write_locked(k.current_key("space-a")).await, + Err(DataKeyError::NotPrepared) + )); + assert!(matches!( + write_locked(k.current_key("space-new")).await, + Err(DataKeyError::NotPrepared) + )); + assert_eq!(rows().await, 0, "nothing is written under the lock"); + + // Reads never create a key: a space with no key stays without one. + k.prepare_read("space-new").await.unwrap(); + assert_eq!(rows().await, 0); + // The lost row comes back from the mirror before the lock. + k.prepare_read("space-a").await.unwrap(); + assert_eq!(rows().await, 1); + assert_eq!( + write_locked(k.key("space-a", DekId::new(1))) + .await + .unwrap() + .bytes(), + a.bytes() + ); +} + #[tokio::test] async fn escrow_rows_or_any_mirror_object_block_provisioning() { let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); @@ -1303,11 +1351,13 @@ async fn a_mirrored_data_key_under_an_unloaded_generation_is_never_replaced() { } #[tokio::test] -async fn a_wrong_holder_can_be_fixed_only_before_the_ceremony_and_any_sealing() { +async fn a_wrong_holder_is_fixed_only_before_the_ceremony_and_with_a_fresh_root_key() { let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); let wrong = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); - let first = wrong.escrow_view().await.unwrap().unwrap().efp; + let old = wrong.escrow_view().await.unwrap().unwrap().bundle; drop(wrong); + let old_keys = at_rest_key::Entity::find().all(&db).await.unwrap(); + let old_escrow = at_rest_escrow::Entity::find().all(&db).await.unwrap(); let fixed = holders(); let allow = |h: &Holders| KeyringConfig { allow_resplit_before_verify: true, @@ -1318,10 +1368,63 @@ async fn a_wrong_holder_can_be_fixed_only_before_the_ceremony_and_any_sealing() open(&db, &mirror, &ROOT, config(&fixed, None)).await, Err(AtRestError::EscrowHoldersChanged) )); - // With it, unverified and nothing sealed: re-split for the new holders. + // With it, unverified and nothing sealed: a fresh ARK, split for the + // new holders. let k = open(&db, &mirror, &ROOT, allow(&fixed)).await.unwrap(); - let second = k.escrow_view().await.unwrap().unwrap().efp; - assert_ne!(second, first); + let new = k.escrow_view().await.unwrap().unwrap().bundle; + assert_ne!(new.fingerprint(), old.fingerprint()); + assert_ne!(new.ark_id, old.ark_id); + // The removed holder's old share, with any other old share, rebuilds + // only the retired ARK, never the one now escrowed. + let old_ark = old + .recover(&h.decrypt(&old, 1), &h.decrypt(&old, 0)) + .unwrap(); + let new_ark = new + .recover(&fixed.decrypt(&new, 0), &fixed.decrypt(&new, 2)) + .unwrap(); + assert_ne!(old_ark.expose(), new_ark.expose()); + // The old ARK's copy, bundle and record objects are gone from the mirror. + let old_hex = hex::encode(old.ark_id.0); + let names = mirror.list(MIRROR_PREFIX).await.unwrap(); + assert!(!names + .iter() + .any(|n| n.starts_with(ARK_MIRROR_PREFIX) && n.contains(&old_hex))); + assert!(!names.contains(&escrow_mirror_name(&old.fingerprint().to_string()))); + drop(k); + + // A database restored to before the re-split never brings the old ARK + // back: its rows are dropped and the fresh ARK comes from the mirror. + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + at_rest_key::Entity::delete_many().exec(&db).await.unwrap(); + at_rest_key::Entity::insert_many(old_keys.into_iter().map(IntoActiveModel::into_active_model)) + .exec(&db) + .await + .unwrap(); + at_rest_escrow::Entity::insert_many( + old_escrow + .into_iter() + .map(IntoActiveModel::into_active_model), + ) + .exec(&db) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&fixed, None)) + .await + .unwrap(); + assert_eq!( + k.escrow_view().await.unwrap().unwrap().bundle.ark_id, + new.ark_id + ); + assert!(at_rest_key::Entity::find() + .all(&db) + .await + .unwrap() + .iter() + .all(|r| r.ark_id == new.ark_id.0)); + // Once verified, the flag no longer helps. verify_escrow(&k, &fixed).await; drop(k); @@ -1329,8 +1432,22 @@ async fn a_wrong_holder_can_be_fixed_only_before_the_ceremony_and_any_sealing() open(&db, &mirror, &ROOT, allow(&h)).await, Err(AtRestError::EscrowHoldersChanged) )); + // Nor when the database lost the record but the mirror still has it. + at_rest_escrow::Entity::update_many() + .col_expr( + at_rest_escrow::Column::Record, + sea_orm::sea_query::Expr::value(Option::::None), + ) + .exec(&db) + .await + .unwrap(); + assert!(matches!( + open(&db, &mirror, &ROOT, allow(&h)).await, + Err(AtRestError::EscrowHoldersChanged) + )); - // Unverified, but a data key exists (data may be sealed): refused. + // Unverified, but a data key exists (data may be sealed): refused, + // also when only the mirror still has it. let (db, mirror) = (self::db().await, Arc::new(MemoryMirror::default())); let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); k.current_key("space-a").await.unwrap(); @@ -1339,6 +1456,41 @@ async fn a_wrong_holder_can_be_fixed_only_before_the_ceremony_and_any_sealing() open(&db, &mirror, &ROOT, allow(&fixed)).await, Err(AtRestError::EscrowHoldersChanged) )); + space_data_key::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + assert!(matches!( + open(&db, &mirror, &ROOT, allow(&fixed)).await, + Err(AtRestError::EscrowHoldersChanged) + )); +} + +#[tokio::test] +async fn a_forged_retirement_marker_never_drops_a_live_root_key() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let bundle = k.escrow_view().await.unwrap().unwrap().bundle; + drop(k); + let id = hex::encode(bundle.ark_id.0); + // A marker for the live ARK under another root's W, and junk. + let forged = WrapKey::derive(&NEW_ROOT).retire_tag(&bundle.ark_id.0); + mirror + .put( + &retired_mirror_name(&id), + serde_json::to_vec(&MirroredRetired::new(id.clone(), &forged)).unwrap(), + ) + .await + .unwrap(); + mirror + .put(&retired_mirror_name("junk"), b"not json".to_vec()) + .await + .unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + assert_eq!( + k.escrow_view().await.unwrap().unwrap().bundle.ark_id, + bundle.ark_id + ); } #[tokio::test] diff --git a/tinycloud-core/src/at_rest/mod.rs b/tinycloud-core/src/at_rest/mod.rs index 27914be0..7b652a5d 100644 --- a/tinycloud-core/src/at_rest/mod.rs +++ b/tinycloud-core/src/at_rest/mod.rs @@ -20,7 +20,7 @@ pub use envelope::{ }; pub use key::{DataKey, DekId, ObjectNameKey}; pub use names::{sealed_object_name, space_tag, PayloadEncoding, SEALED_NAME_PREFIX}; -pub use provider::{DataKeyError, DataKeyProvider}; +pub use provider::{in_write_lock, write_locked, DataKeyError, DataKeyProvider}; pub use stream::{DecryptReader, EncryptReader, SegmentRange}; #[cfg(any(test, feature = "test-util"))] pub use testing::FixedKeyProvider; diff --git a/tinycloud-core/src/at_rest/provider.rs b/tinycloud-core/src/at_rest/provider.rs index a5273614..c35bf08c 100644 --- a/tinycloud-core/src/at_rest/provider.rs +++ b/tinycloud-core/src/at_rest/provider.rs @@ -1,6 +1,8 @@ //! The boundary between key continuity (who holds DEKs) and the envelope //! (what a DEK encrypts). See `docs/at-rest/INTERFACES.md` ยง2. +use std::future::Future; + use super::key::{DataKey, DekId, ObjectNameKey}; use async_trait::async_trait; @@ -16,10 +18,34 @@ pub trait DataKeyProvider: Send + Sync { /// The DEK a stored blob names in its header. Never creates a key. async fn key(&self, space: &str, dek_id: DekId) -> Result; + /// Before work that may read `space`'s sealed blobs under the invoke + /// write lock: put back any of the space's keys the database lost (for + /// example from an authenticated mirror). Never creates a key. Inside + /// [`write_locked`], `key` and `current_key` only read. + async fn prepare_read(&self, space: &str) -> Result<(), DataKeyError>; + /// The object-name key for sealed S3 names. Constant for a deployment. fn object_name_key(&self) -> Result; } +tokio::task_local! { + static WRITE_LOCKED: (); +} + +/// Runs `f` as work under the invoke write lock and transaction. Key +/// providers make no key-store writes there (on SQLite they would wait on the +/// lock this task holds): a key that needs creating or restoring answers +/// [`DataKeyError::NotPrepared`] instead. Callers prepare spaces first +/// (`current_key` for written spaces, `prepare_read` for read ones). +pub async fn write_locked(f: F) -> F::Output { + WRITE_LOCKED.scope((), f).await +} + +/// Whether the current task is inside [`write_locked`]. +pub fn in_write_lock() -> bool { + WRITE_LOCKED.try_with(|_| ()).is_ok() +} + #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum DataKeyError { @@ -33,4 +59,8 @@ pub enum DataKeyError { /// Storage or unwrap failure. Carries no key bytes. #[error("data key backend: {0}")] Backend(#[source] Box), + /// The key needed creating or restoring inside [`write_locked`], where + /// no key-store write may happen: the space was not prepared first. + #[error("the space's data key was not prepared before the write lock")] + NotPrepared, } diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs index 8a945ca5..de615e30 100644 --- a/tinycloud-core/src/at_rest/store.rs +++ b/tinycloud-core/src/at_rest/store.rs @@ -39,10 +39,47 @@ pub trait KeyMirror: Send + Sync { async fn get(&self, name: &str) -> Result>, MirrorError>; /// Names under `prefix`, in any order. async fn list(&self, prefix: &str) -> Result, MirrorError>; + /// Remove an object; absent is fine. Only used to retire key material + /// that never protected anything. + async fn delete(&self, name: &str) -> Result<(), MirrorError>; } pub const MIRROR_PREFIX: &str = "_tc/at-rest/"; pub(crate) const ARK_MIRROR_PREFIX: &str = "_tc/at-rest/ark-"; +pub(crate) const RETIRED_MIRROR_PREFIX: &str = "_tc/at-rest/retired-"; + +/// A W-authenticated marker that an ARK was retired (see +/// `WrapKey::retire_tag`). +pub(crate) fn retired_mirror_name(ark_id_hex: &str) -> String { + format!("{RETIRED_MIRROR_PREFIX}{ark_id_hex}.json") +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct MirroredRetired { + pub version: u32, + pub ark_id: String, + pub tag: String, +} + +impl MirroredRetired { + pub fn new(ark_id_hex: String, tag: &[u8]) -> Self { + Self { + version: MIRROR_VERSION, + ark_id: ark_id_hex, + tag: hex::encode(tag), + } + } + + /// `(ark_id, tag)` bytes, if well formed and a version this node reads. + pub fn parts(&self) -> Option<(Vec, Vec)> { + check_version(self.version).ok()?; + Some(( + hex::decode(&self.ark_id).ok()?, + hex::decode(&self.tag).ok()?, + )) + } +} pub(crate) fn ark_mirror_name(generation: u32, ark_id_hex: &str) -> String { format!("{ARK_MIRROR_PREFIX}{generation:010}-{ark_id_hex}.json") @@ -307,10 +344,53 @@ impl AtRestStore { Ok(at_rest_escrow::Entity::find().count(&self.db).await? > 0) } + /// Drop every key and escrow row and store `row` as generation 1, in + /// one transaction. Only for replacing an ARK that never protected + /// anything (no data key, never verified). + pub async fn replace_all_keys(&self, row: at_rest_key::Model) -> Result<(), DbErr> { + let _w = self.writer().await; + let tx = self.db.begin().await?; + at_rest_escrow::Entity::delete_many().exec(&tx).await?; + at_rest_key::Entity::delete_many().exec(&tx).await?; + at_rest_key::Entity::insert(at_rest_key::ActiveModel { + generation: Set(row.generation), + ark_id: Set(row.ark_id), + w_label: Set(row.w_label), + sealed: Set(row.sealed), + chain: Set(row.chain), + created_at: Set(row.created_at), + }) + .exec(&tx) + .await?; + tx.commit().await + } + + /// Remove key rows of retired ARKs and every escrow row, in one + /// transaction (escrow rows of a retired ARK must never load either). + pub async fn drop_retired_keys(&self, ark_ids: &[Vec]) -> Result<(), DbErr> { + let _w = self.writer().await; + let tx = self.db.begin().await?; + at_rest_escrow::Entity::delete_many().exec(&tx).await?; + at_rest_key::Entity::delete_many() + .filter(at_rest_key::Column::ArkId.is_in(ark_ids.iter().cloned())) + .exec(&tx) + .await?; + tx.commit().await + } + pub async fn has_space_keys(&self) -> Result { Ok(space_data_key::Entity::find().count(&self.db).await? > 0) } + /// Any escrow row holding a ceremony record, verifiable or not. + pub async fn has_escrow_record(&self) -> Result { + Ok(at_rest_escrow::Entity::find() + .filter(at_rest_escrow::Column::Record.is_not_null()) + .count(&self.db) + .await? + > 0) + } + /// The generation's active bundle. The partial unique index and /// `activate_escrow`'s lock keep it unique; the order makes the choice /// deterministic even if a database lacks the index. @@ -523,4 +603,13 @@ impl KeyMirror for MemoryMirror { .cloned() .collect()) } + + async fn delete(&self, name: &str) -> Result<(), MirrorError> { + self.check()?; + if self.read_only.load(std::sync::atomic::Ordering::SeqCst) { + return Err("mirror is read-only".into()); + } + self.objects.lock().remove(name); + Ok(()) + } } diff --git a/tinycloud-core/src/at_rest/testing.rs b/tinycloud-core/src/at_rest/testing.rs index ef7054a8..f023ad24 100644 --- a/tinycloud-core/src/at_rest/testing.rs +++ b/tinycloud-core/src/at_rest/testing.rs @@ -79,6 +79,10 @@ impl DataKeyProvider for FixedKeyProvider { )) } + async fn prepare_read(&self, _space: &str) -> Result<(), DataKeyError> { + Ok(()) + } + fn object_name_key(&self) -> Result { if self.locked { return Err(DataKeyError::Locked); From 3befa10b70bff83b3efaa0cda0d5635bf7183ad2 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 16:00:06 +0000 Subject: [PATCH 13/15] chore(TC-887): log the data-key object name when a mirrored DEK fails closed (#326) --- tinycloud-core/src/at_rest/keyring.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index 9595a6bb..14631aa1 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -1575,7 +1575,10 @@ impl AtRestKeyring { let newest = keys.newest().0; let names_unloaded_generation = serde_json::from_slice::(&bytes).is_ok_and(|m| m.generation > newest); + // The object name is a keyed hash, not the space: safe to log, + // and what an operator needs for the dek_unavailable runbook. tracing::error!( + object = %name, dek_id, names_unloaded_generation, "a mirrored data key for this space does not authenticate; refusing to make a new one" @@ -1618,9 +1621,16 @@ impl AtRestKeyring { Some((m, _)) if m.generation == row.generation && m.wrapped == row.wrapped => { Ok(()) } - _ => Err(backend(AtRestError::DekUnavailable( - "the mirror holds different bytes for this data key; it is never overwritten", - ))), + _ => { + tracing::error!( + object = %name, + dek_id, + "the mirror holds different bytes for this data key; refusing to overwrite it" + ); + Err(backend(AtRestError::DekUnavailable( + "the mirror holds different bytes for this data key; it is never overwritten", + ))) + } }; } let body = serde_json::to_vec(&MirroredDek::from_row(row)).expect("serializable"); From 01055c7a7181d06c0723dc390651fb8b10e02fd4 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 16:51:43 +0000 Subject: [PATCH 14/15] fix(TC-887): an existing ARK never gets a second bundle for other holders (#326) When no active bundle exists and none can be restored, establish_escrow used to split the ARK again for whatever holders were configured. Now any evidence of an earlier bundle for this ARK with other holders refuses the boot (EscrowHoldersChanged), strongest first: - a stored escrow row for this ARK; - a mirrored ceremony record whose MAC verifies under the ARK against its mirrored bundle (a record that doesn't verify proves nothing); - a mirrored bundle object naming this ARK: unauthenticated, so it can only refuse, never be used. A planted one fails closed (like D1); the log names the object, the runbook is to verify it and remove it. The operator route is the same as for an active bundle: allow_resplit_before_verify (a fresh ARK) before the ceremony with nothing sealed, an ARK rotation (TC-1345) after it. may_resplit no longer takes the active bundle (any stored or mirrored record refuses). Tests: rows lost + holders changed refuses without the flag and gives a fresh ARK with it; after the ceremony the verified record refuses even with the flag; a planted bundle only refuses and removing it boots. --- tinycloud-core/src/at_rest/keyring.rs | 166 +++++++++++++++----- tinycloud-core/src/at_rest/keyring/tests.rs | 98 ++++++++++++ tinycloud-core/src/at_rest/store.rs | 5 + 3 files changed, 233 insertions(+), 36 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index 14631aa1..02786416 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -77,7 +77,7 @@ pub enum AtRestError { Conflict(&'static str), #[error("at_rest.escrow.verified_fp pins an escrow that is not for the configured recipients and the newest key; the node never re-splits while a fingerprint is pinned")] EscrowPinned, - #[error("the configured escrow recipients differ from the active escrow bundle's; the node never re-splits on its own (before the ceremony, at_rest.escrow.allow_resplit_before_verify allows it; afterwards an explicit re-escrow is needed)")] + #[error("the configured escrow recipients differ from an existing escrow for this ARK, and an ARK never gets a second split for other holders. Before the ceremony with nothing sealed: set at_rest.escrow.allow_resplit_before_verify (the node makes a fresh ARK). After it: an ARK rotation (TC-1345). If the log names an unexpected block-store object, verify it is not a genuine escrow, then remove it")] EscrowHoldersChanged, #[error("a space's data key is unavailable: {0}")] DekUnavailable(&'static str), @@ -827,15 +827,23 @@ impl AtRestKeyring { /// holders. This is the only place a bundle is ever created or /// activated; request paths only read ([`Self::active_bundle`]). /// - /// - An active bundle for other holders: refused - /// ([`AtRestError::EscrowHoldersChanged`]); the node never re-splits on - /// its own. + /// An existing ARK never gets a second bundle for other holders: + /// + /// - An active bundle for other holders refuses the boot + /// ([`AtRestError::EscrowHoldersChanged`]). /// - None active and `verified_fp` pinned: the pinned bundle, from the /// database or the mirror (the pin authenticates it), is re-activated; /// anything else is [`AtRestError::EscrowPinned`]. /// - None active, no pin: a mirrored bundle whose mirrored ceremony /// record verifies under the ARK is restored (a database restore - /// doesn't force a new ceremony); otherwise a new bundle is made. + /// doesn't force a new ceremony). Otherwise a new bundle is made, + /// unless any evidence of an earlier bundle for this ARK with other + /// holders exists ([`Self::other_holders_evidence`]): that refuses + /// the boot too. + /// + /// A refused holder change can be fixed only before the ceremony, with + /// `allow_resplit_before_verify` ([`Self::may_resplit`]), which + /// replaces the ARK; afterwards it needs a rotation (TC-1345). /// /// A verification record the database lost is restored from the mirror /// when it verifies under the ARK. @@ -859,16 +867,9 @@ impl AtRestKeyring { } return Ok(()); } - if !self.may_resplit(&bundle).await? { - return Err(AtRestError::EscrowHoldersChanged); - } - tracing::warn!( - generation = g, - "replacing an unverified root key for corrected holders (allow_resplit_before_verify): the old shares are worthless and the ceremony starts over" - ); - self.retire_and_replace().await?; - // The fresh key has no escrow yet: this makes its bundle. - return Box::pin(self.establish_escrow()).await; + return self + .holders_changed(g, "the active escrow bundle".into()) + .await; } let wanted = |b: &EscrowBundle| { b.holders() == self.config.holders && b.ark_id == gen.ark_id && b.generation == g @@ -885,16 +886,21 @@ impl AtRestKeyring { }; let (bundle, created) = match restored { Some(bundle) => (bundle, false), - None => ( - EscrowBundle::create( - &gen.ark, - gen.ark_id, - g, - &self.config.holders, - self.config.recipient_policy, - )?, - true, - ), + None => { + if let Some(evidence) = self.other_holders_evidence(&gen.ark, gen.ark_id).await? { + return self.holders_changed(g, evidence).await; + } + ( + EscrowBundle::create( + &gen.ark, + gen.ark_id, + g, + &self.config.holders, + self.config.recipient_policy, + )?, + true, + ) + } }; let row = at_rest_escrow::Model { efp: bundle.fingerprint().to_string(), @@ -931,20 +937,108 @@ impl AtRestKeyring { Ok(()) } - /// A holder change may re-split only when explicitly allowed, nothing is - /// pinned, no ceremony record exists, and no data key was ever made. - /// The database and the mirror are both checked (a database restored to - /// an earlier point may have lost a data key or record the mirror still - /// has). Any record or data-key object in the mirror counts, readable - /// or not: it refuses only this pre-ceremony shortcut. - async fn may_resplit(&self, active: &EscrowBundle) -> Result { + /// The configured holders differ from an existing bundle for the ARK. + /// Refused, unless [`Self::may_resplit`] allows replacing the ARK. + async fn holders_changed(&self, g: u32, evidence: String) -> Result<(), AtRestError> { + if !self.may_resplit().await? { + tracing::error!( + generation = g, + %evidence, + "the configured escrow holders differ from an existing escrow for this ARK; refusing to split it again. Before the ceremony: allow_resplit_before_verify (a fresh ARK). After it: an ARK rotation (TC-1345). An unexpected mirror object: verify it, then remove it" + ); + return Err(AtRestError::EscrowHoldersChanged); + } + tracing::warn!( + generation = g, + %evidence, + "replacing an unverified root key for corrected holders (allow_resplit_before_verify): the old shares are worthless and the ceremony starts over" + ); + self.retire_and_replace().await?; + // The fresh key has no escrow yet: this makes its bundle. + Box::pin(self.establish_escrow()).await + } + + /// Evidence that `ark_id` already has a bundle for holders other than + /// the configured ones, strongest first: + /// + /// 1. a database escrow row for this ARK; + /// 2. a mirrored ceremony record whose MAC verifies under the ARK + /// against its mirrored bundle, for other holders (authenticated); + /// a record that doesn't verify proves nothing and is skipped; + /// 3. a mirrored bundle object naming this ARK. Unauthenticated: it can + /// only refuse, never be used. + /// + /// Evidence for the configured holders themselves is not a change. + async fn other_holders_evidence( + &self, + ark: &Ark, + ark_id: ArkId, + ) -> Result, AtRestError> { + for row in self.store.escrows().await? { + let bundle = Self::parse_bundle(&row)?; + if bundle.ark_id == ark_id && bundle.holders() != self.config.holders { + return Ok(Some(format!("stored escrow bundle {}", row.efp))); + } + } + let records = self + .mirror + .list(&format!("{MIRROR_PREFIX}record-")) + .await + .map_err(mirror_err)?; + for name in records { + let Some(bytes) = get_capped(&*self.mirror, &name).await.map_err(mirror_err)? else { + continue; + }; + let Ok(record) = VerificationRecord::from_json(&bytes) else { + continue; + }; + if record.ark_id != ark_id { + continue; + } + let bundle = get_capped(&*self.mirror, &escrow_mirror_name(&record.efp.to_string())) + .await + .map_err(mirror_err)? + .and_then(|b| EscrowBundle::from_json(&b).ok()); + if let Some(b) = bundle { + if record.verify(ark, &b).is_ok() && b.holders() != self.config.holders { + return Ok(Some(format!( + "verified ceremony record {name}, for other holders" + ))); + } + } + } + let bundles = self + .mirror + .list(&format!("{MIRROR_PREFIX}escrow-")) + .await + .map_err(mirror_err)?; + for name in bundles { + let Some(bytes) = get_capped(&*self.mirror, &name).await.map_err(mirror_err)? else { + continue; + }; + let Ok(bundle) = EscrowBundle::from_json(&bytes) else { + continue; + }; + if bundle.ark_id == ark_id && bundle.holders() != self.config.holders { + return Ok(Some(format!( + "unauthenticated block-store object {name}, a bundle for this ARK with other holders" + ))); + } + } + Ok(None) + } + + /// A holder change may replace the ARK only when explicitly allowed, + /// nothing is pinned, no ceremony record exists, and no data key was + /// ever made. The database and the mirror are both checked (a database + /// restored to an earlier point may have lost a data key or record the + /// mirror still has). Any record or data-key object in the mirror + /// counts, readable or not: it refuses only this pre-ceremony shortcut. + async fn may_resplit(&self) -> Result { if !self.config.allow_resplit_before_verify || self.config.pinned_efp.is_some() { return Ok(false); } - if self.verified_record(active).await?.is_some() - || self.store.has_space_keys().await? - || self.store.has_escrow_record().await? - { + if self.store.has_space_keys().await? || self.store.has_escrow_record().await? { return Ok(false); } for prefix in ["record-", "dek-"] { diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index ee54c9c6..c431a10d 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -1466,6 +1466,104 @@ async fn a_wrong_holder_is_fixed_only_before_the_ceremony_and_with_a_fresh_root_ )); } +/// The database lost its escrow rows and the configured holders changed: +/// the ARK must not get a second split for other holders. +#[tokio::test] +async fn an_ark_never_gets_a_second_bundle_for_other_holders() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let old = k.escrow_view().await.unwrap().unwrap().bundle; + drop(k); + let escrow_rows = || async { at_rest_escrow::Entity::find().all(&db).await.unwrap() }; + let lose_escrow_rows = || async { + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + }; + let fixed = holders(); + let allow = |h: &Holders| KeyringConfig { + allow_resplit_before_verify: true, + ..config(h, None) + }; + + // Before the ceremony: the mirrored bundle object is the evidence. + lose_escrow_rows().await; + assert!(matches!( + open(&db, &mirror, &ROOT, config(&fixed, None)).await, + Err(AtRestError::EscrowHoldersChanged) + )); + assert!(escrow_rows().await.is_empty(), "nothing was split"); + // With the flag (unverified, nothing sealed): a fresh ARK instead. + let k = open(&db, &mirror, &ROOT, allow(&fixed)).await.unwrap(); + let new = k.escrow_view().await.unwrap().unwrap().bundle; + assert_ne!(new.ark_id, old.ark_id); + assert_eq!(new.holders(), fixed.recipients); + + // After the ceremony: the verified record is the evidence, and the flag + // no longer helps. + verify_escrow(&k, &fixed).await; + drop(k); + lose_escrow_rows().await; + for config in [config(&h, None), allow(&h)] { + assert!(matches!( + open(&db, &mirror, &ROOT, config).await, + Err(AtRestError::EscrowHoldersChanged) + )); + } + assert!(escrow_rows().await.is_empty()); + // The configured holders themselves: the verified bundle comes back. + let k = open(&db, &mirror, &ROOT, config(&fixed, None)) + .await + .unwrap(); + assert_eq!( + k.escrow_view().await.unwrap().unwrap().bundle.fingerprint(), + new.fingerprint() + ); +} + +/// A bundle object planted in the bucket for the live ARK can only make a +/// boot that lost its escrow rows refuse (fail closed, like D1); removing +/// it lets the node boot. +#[tokio::test] +async fn a_planted_bundle_only_refuses_and_is_never_used() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let real = k.escrow_view().await.unwrap().unwrap().bundle; + drop(k); + let other = holders(); + let planted = EscrowBundle::create( + &Ark::generate(), + real.ark_id, + real.generation, + &other.recipients, + RecipientPolicy::AllowSoftware, + ) + .unwrap(); + let planted_name = escrow_mirror_name(&planted.fingerprint().to_string()); + mirror + .put(&planted_name, serde_json::to_vec(&planted).unwrap()) + .await + .unwrap(); + // A healthy database: the planted object changes nothing. + open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + // Escrow rows lost: refused, never split for the planted holders. + at_rest_escrow::Entity::delete_many() + .exec(&db) + .await + .unwrap(); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::EscrowHoldersChanged) + )); + // Runbook: verify the object isn't a genuine escrow, then remove it. + mirror.delete(&planted_name).await.unwrap(); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let bundle = k.escrow_view().await.unwrap().unwrap().bundle; + assert_eq!(bundle.ark_id, real.ark_id); + assert_eq!(bundle.holders(), h.recipients); +} + #[tokio::test] async fn a_forged_retirement_marker_never_drops_a_live_root_key() { let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); diff --git a/tinycloud-core/src/at_rest/store.rs b/tinycloud-core/src/at_rest/store.rs index de615e30..661ee9f8 100644 --- a/tinycloud-core/src/at_rest/store.rs +++ b/tinycloud-core/src/at_rest/store.rs @@ -413,6 +413,11 @@ impl AtRestStore { .await } + /// Every escrow row, active or not. + pub async fn escrows(&self) -> Result, DbErr> { + at_rest_escrow::Entity::find().all(&self.db).await + } + /// Make `row` the generation's only active bundle. Older bundles stay. /// /// Concurrent activations serialize on the generation's `at_rest_key` From 9131d244fa205d645fd0e4eb38072a2612bb2f8a Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 18:41:20 +0000 Subject: [PATCH 15/15] fix(TC-887): an unreadable retirement marker never stops a node in mode off (#326) Review 761d4859: one unreadable retired-.json, even for an ARK that exists nowhere, failed every boot, mode off and healthy database included. A marker that can't be read is now skipped with a warning unless its name is for a stored or mirrored ARK and keys may seal (require_mirror); only then could skipping it bring a retired key back, so the boot fails. load() reads the stored and mirrored rows first to know those ids. Test from the review's probe (RED before this change). --- tinycloud-core/src/at_rest/keyring.rs | 36 +++++++++++++++++---- tinycloud-core/src/at_rest/keyring/tests.rs | 31 ++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/tinycloud-core/src/at_rest/keyring.rs b/tinycloud-core/src/at_rest/keyring.rs index 02786416..c3fc733f 100644 --- a/tinycloud-core/src/at_rest/keyring.rs +++ b/tinycloud-core/src/at_rest/keyring.rs @@ -653,8 +653,14 @@ impl AtRestKeyring { async fn load(&self) -> Result<(), AtRestError> { // A retired ARK (W-authenticated tombstone in the mirror) is never // loaded, even from a database restored to before its retirement. - let retired = self.retired_ark_ids().await?; let mut db = self.store.keys().await?; + let mut mirrored = mirrored_rows(&*self.mirror, !db.is_empty()).await?; + let known: Vec<&[u8]> = db + .iter() + .chain(&mirrored) + .map(|r| r.ark_id.as_slice()) + .collect(); + let retired = self.retired_ark_ids(&known).await?; if db.iter().any(|r| retired.contains(&r.ark_id)) { // Retirement replaces every generation at once, so a database // holding both kinds was not written by this node. @@ -668,12 +674,12 @@ impl AtRestKeyring { tracing::warn!("dropping key and escrow rows of a retired at-rest root key"); self.store.drop_retired_keys(&retired).await?; db = self.store.keys().await?; + mirrored = mirrored_rows(&*self.mirror, !db.is_empty()).await?; } if !retired.is_empty() { self.sweep_retired(&retired).await?; } check_contiguous(&db)?; - let mut mirrored = mirrored_rows(&*self.mirror, !db.is_empty()).await?; mirrored.retain(|r| !retired.contains(&r.ark_id)); if db.is_empty() && mirrored.is_empty() { let live_mirror_objects = self @@ -1097,9 +1103,12 @@ impl AtRestKeyring { } /// ARK ids with a tombstone that authenticates under this node's W. - /// Junk or forged tombstones are skipped; a read error fails the call - /// (skipping could bring a retired ARK back). - async fn retired_ark_ids(&self) -> Result>, AtRestError> { + /// Junk or forged tombstones are skipped with a warning. A tombstone + /// that can't be read is skipped too, unless its name is for an ARK + /// in `known` (stored or mirrored) and keys may seal (`require_mirror`): + /// then skipping could bring a retired ARK back, so the boot fails. + /// With mode off, mirror contents never stop the node. + async fn retired_ark_ids(&self, known: &[&[u8]]) -> Result>, AtRestError> { let mut ids = Vec::new(); let names = self .mirror @@ -1107,8 +1116,21 @@ impl AtRestKeyring { .await .map_err(mirror_err)?; for name in names { - let Some(bytes) = get_capped(&*self.mirror, &name).await.map_err(mirror_err)? else { - continue; + let bytes = match get_capped(&*self.mirror, &name).await { + Ok(Some(bytes)) => bytes, + Ok(None) => continue, + Err(error) => { + let names_known = name + .strip_prefix(RETIRED_MIRROR_PREFIX) + .and_then(|rest| rest.strip_suffix(".json")) + .and_then(|id| hex::decode(id).ok()) + .is_some_and(|id| known.contains(&id.as_slice())); + if names_known && self.config.require_mirror { + return Err(mirror_err(error)); + } + tracing::warn!(object = ?name, %error, names_known, "ignoring an at-rest retirement marker that can't be read"); + continue; + } }; let parts = serde_json::from_slice::(&bytes) .ok() diff --git a/tinycloud-core/src/at_rest/keyring/tests.rs b/tinycloud-core/src/at_rest/keyring/tests.rs index c431a10d..d2029447 100644 --- a/tinycloud-core/src/at_rest/keyring/tests.rs +++ b/tinycloud-core/src/at_rest/keyring/tests.rs @@ -1564,6 +1564,37 @@ async fn a_planted_bundle_only_refuses_and_is_never_used() { assert_eq!(bundle.holders(), h.recipients); } +/// Review 761d4859: one unreadable retirement marker must not stop every +/// node. It blocks only when it names a known ARK and keys may seal; +/// with mode off, mirror contents never stop the node. +#[tokio::test] +async fn an_unreadable_retirement_marker_blocks_only_a_known_key_once_keys_may_seal() { + let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders()); + let k = open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + let live = k.escrow_view().await.unwrap().unwrap().bundle.ark_id; + drop(k); + let mode_off = |h: &Holders| KeyringConfig { + require_mirror: false, + ..config(h, None) + }; + + // For an ARK that exists nowhere: skipped, with or without sealing. + mirror.plant_unreadable(&retired_mirror_name(&hex::encode([0xab; 16]))); + open(&db, &mirror, &ROOT, config(&h, None)).await.unwrap(); + open(&db, &mirror, &ROOT, mode_off(&h)).await.unwrap(); + + // For the live ARK: mode off still boots on the same key; once keys + // may seal, skipping could bring a retired key back, so it fails. + mirror.plant_unreadable(&retired_mirror_name(&hex::encode(live.0))); + let k = open(&db, &mirror, &ROOT, mode_off(&h)).await.unwrap(); + assert_eq!(k.escrow_view().await.unwrap().unwrap().bundle.ark_id, live); + drop(k); + assert!(matches!( + open(&db, &mirror, &ROOT, config(&h, None)).await, + Err(AtRestError::Mirror(_)) + )); +} + #[tokio::test] async fn a_forged_retirement_marker_never_drops_a_live_root_key() { let (db, mirror, h) = (db().await, Arc::new(MemoryMirror::default()), holders());