From bdc9b59af1db63f814a82bd3fd75635ea477cb70 Mon Sep 17 00:00:00 2001 From: Jason Hernandez <7144515+jasonhernandez@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:15:48 -0700 Subject: [PATCH] deps: switch reqwest from native-tls to rustls with aws-lc-rs Build reqwest 0.13 with its `rustls` feature (aws-lc-rs provider, platform verifier) instead of `native-tls-vendored`, and switch sentry from `transport` to `reqwest` + `rustls`. Sentry's `transport` feature enables `reqwest/native-tls-no-alpn`, and reqwest defaults to native-tls whenever it is compiled in, so leaving it would keep every reqwest 0.13 client on OpenSSL. The schema registry client (mz-ccsr) used native-tls-only APIs: * `Identity` now holds a PEM key and certificate chain instead of a PKCS #12 archive. `Identity::from_pem` checks that the key matches the leaf certificate up front, and the buffer is zeroized on drop. `Debug` no longer prints the key. * `Identity::from_pkcs12_der` is removed. testdrive, its only caller, converts its PKCS #12 keystore to PEM with OpenSSL. * `Certificate` parsing uses rustls instead of native-tls. mz-ccsr no longer depends on native-tls, openssl or mz-tls-util, and mz-storage-types drops its native-tls and openssl dependencies. The kafka-auth schema registry tests expect the rustls error strings. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 26 +- Cargo.toml | 4 +- misc/wasm/Cargo.lock | 448 ++++++++++++++++-- src/ccsr/Cargo.toml | 6 +- src/ccsr/src/tls.rs | 108 ++++- src/ccsr/tests/client.rs | 56 ++- src/storage-types/Cargo.toml | 2 - src/storage-types/src/errors.rs | 4 +- src/testdrive/Cargo.toml | 1 + src/testdrive/src/action.rs | 30 +- test/kafka-auth/test-schema-registry-mssl.td | 5 +- .../test-schema-registry-ssl-basic.td | 10 +- test/kafka-auth/test-schema-registry-ssl.td | 2 +- 13 files changed, 599 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24f456a09206e..f850bc2032d6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6594,15 +6594,15 @@ dependencies = [ "hyper-util", "mz-build-tools", "mz-ore", - "mz-tls-util", - "native-tls", "openssl", "proptest", "proptest-derive", "prost-build", "reqwest 0.13.5", + "rustls", "serde", "serde_json", + "thiserror 2.0.21", "tokio", "tracing", "url", @@ -8922,8 +8922,6 @@ dependencies = [ "mz-timely-util", "mz-tls-util", "mz-tracing", - "native-tls", - "openssl", "proptest", "proptest-derive", "prost", @@ -9007,6 +9005,7 @@ dependencies = [ "mz-sql-server-util", "mz-storage-types", "mz-tls-util", + "openssl", "parquet", "postgres_array", "prost", @@ -11669,12 +11668,10 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", "mime", - "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -11686,7 +11683,6 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower 0.5.3", @@ -12290,8 +12286,8 @@ checksum = "9fa951f4e644464ebfd2347a7ab03b0828c512d2448a76262cad2607b343dff4" dependencies = [ "cfg_aliases", "httpdate", - "native-tls", "reqwest 0.13.5", + "rustls", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -14415,14 +14411,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a" dependencies = [ "base64 0.22.1", - "der", "log", - "native-tls", "percent-encoding", + "rustls", "rustls-pki-types", "ureq-proto", "utf-8", - "webpki-root-certs", + "webpki-roots", ] [[package]] @@ -14701,6 +14696,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "8.0.6" diff --git a/Cargo.toml b/Cargo.toml index c6e042d322183..988b23daadc3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -467,7 +467,7 @@ rdkafka-sys = { version = "4.3.0", features = ["cmake-build", "libz-static", "ss regex = "1.12.3" regex-syntax = "0.8.10" reqsign-core = "3.3.0" -reqwest = { version = "0.13.5", default-features = false, features = ["blocking", "charset", "cookies", "http2", "json", "native-tls-vendored", "query", "stream", "system-proxy"] } +reqwest = { version = "0.13.5", default-features = false, features = ["blocking", "charset", "cookies", "http2", "json", "query", "rustls", "stream", "system-proxy"] } # The iceberg fork's public API (`with_client`, `OAuth2TokenProvider`) takes # reqwest 0.12 types, so the iceberg connection code in mz-storage-types uses # this alias. @@ -486,7 +486,7 @@ scopeguard = "1.2.0" seahash = "4.1.0" segment = { version = "0.2.6", default-features = false, features = ["native-tls-vendored"] } semver = "1.0.28" -sentry = { version = "0.49.1", default-features = false, features = ["backtrace", "contexts", "debug-images", "transport"] } +sentry = { version = "0.49.1", default-features = false, features = ["backtrace", "contexts", "debug-images", "reqwest", "rustls"] } sentry-panic = "0.49.1" sentry-tracing = "0.49.1" serde = { version = "1.0.219", features = ["derive"] } diff --git a/misc/wasm/Cargo.lock b/misc/wasm/Cargo.lock index 635192fb51a14..b9e5f125b9f45 100644 --- a/misc/wasm/Cargo.lock +++ b/misc/wasm/Cargo.lock @@ -70,6 +70,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-lc-rs" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "axum" version = "0.8.9" @@ -140,12 +163,6 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" -[[package]] -name = "base64ct" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - [[package]] name = "bitflags" version = "2.11.0" @@ -172,11 +189,13 @@ checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" [[package]] name = "cc" -version = "1.2.41" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -203,6 +222,25 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -257,16 +295,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "pem-rfc7468", - "zeroize", -] - [[package]] name = "deranged" version = "0.5.8" @@ -295,6 +323,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.18.0" @@ -336,9 +370,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "findshlibs" @@ -393,6 +427,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.31" @@ -465,6 +505,19 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.3.3" @@ -484,9 +537,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -604,6 +659,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -836,6 +906,65 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.21", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.105" @@ -908,6 +1037,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "matchers" version = "0.2.0" @@ -1223,9 +1358,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl" @@ -1388,15 +1523,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1608,6 +1734,63 @@ dependencies = [ "cc", ] +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -1675,6 +1858,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.12" @@ -1715,11 +1907,11 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-core", @@ -1728,19 +1920,21 @@ dependencies = [ "http-body", "http-body-util", "hyper", - "hyper-tls", + "hyper-rustls", "hyper-util", "js-sys", "log", - "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -1750,12 +1944,32 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1778,15 +1992,83 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.19" @@ -1799,6 +2081,15 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -1851,8 +2142,8 @@ checksum = "9fa951f4e644464ebfd2347a7ab03b0828c512d2448a76262cad2607b343dff4" dependencies = [ "cfg_aliases", "httpdate", - "native-tls", "reqwest", + "rustls", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -1992,9 +2283,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simd-adler32" @@ -2002,6 +2293,22 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.1" @@ -2071,6 +2378,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "1.0.109" @@ -2226,6 +2539,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tokio" version = "1.45.0" @@ -2263,6 +2582,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -2520,6 +2849,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "ureq" version = "3.2.0" @@ -2527,14 +2862,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc" dependencies = [ "base64 0.22.1", - "der", "log", - "native-tls", "percent-encoding", + "rustls", "rustls-pki-types", "ureq-proto", "utf-8", - "webpki-root-certs", + "webpki-roots", ] [[package]] @@ -2607,6 +2941,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2715,6 +3059,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2731,6 +3084,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/src/ccsr/Cargo.toml b/src/ccsr/Cargo.toml index 99cc0291a80ba..b54ea464d184a 100644 --- a/src/ccsr/Cargo.toml +++ b/src/ccsr/Cargo.toml @@ -11,20 +11,20 @@ workspace = true [dependencies] anyhow.workspace = true -native-tls.workspace = true -openssl.workspace = true reqwest.workspace = true -mz-tls-util = { path = "../tls-util" } proptest.workspace = true zeroize.workspace = true proptest-derive.workspace = true +rustls.workspace = true serde.workspace = true +thiserror.workspace = true url = { workspace = true, features = ["serde"] } [dev-dependencies] hyper.workspace = true hyper-util.workspace = true mz-ore = { path = "../ore", features = ["async", "test"] } +openssl.workspace = true serde_json.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/src/ccsr/src/tls.rs b/src/ccsr/src/tls.rs index 09369a681c891..cbcc9cf55329a 100644 --- a/src/ccsr/src/tls.rs +++ b/src/ccsr/src/tls.rs @@ -9,24 +9,51 @@ //! TLS certificates and identities. +use std::fmt; +use std::sync::Arc; + +use rustls::pki_types::pem::PemObject; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use serde::{Deserialize, Serialize}; +use zeroize::{Zeroize, Zeroizing}; -use mz_tls_util::pkcs12der_from_pem; -use zeroize::Zeroize; +/// An error constructing a [`Certificate`] or [`Identity`]. +#[derive(Debug, thiserror::Error)] +pub enum TlsError { + #[error("invalid PEM: {0}")] + Pem(#[from] rustls::pki_types::pem::Error), + #[error("no certificate found in PEM input")] + NoCertificate, + #[error("no private key found in PEM input")] + NoPrivateKey, + #[error("invalid certificate: {0}")] + Certificate(rustls::CertificateError), + #[error("invalid TLS identity: {0}")] + Identity(rustls::Error), + #[error(transparent)] + Reqwest(#[from] reqwest::Error), +} /// A [Serde][serde]-enabled wrapper around [`reqwest::Identity`]. /// +/// Holds the PEM-encoded private key and certificate chain. The buffer is +/// zeroized on drop. +/// /// [Serde]: serde -#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[derive(Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct Identity { - der: Vec, - pass: String, + pem: Vec, +} + +impl fmt::Debug for Identity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Identity").finish_non_exhaustive() + } } impl Zeroize for Identity { fn zeroize(&mut self) { - self.der.zeroize(); - self.pass.zeroize(); + self.pem.zeroize(); } } @@ -37,22 +64,42 @@ impl Drop for Identity { } impl Identity { - /// Constructs an identity from a PEM-formatted key and certificate using OpenSSL. - pub fn from_pem(key: &[u8], cert: &[u8]) -> Result { - let (der, pass) = pkcs12der_from_pem(key, cert)?.into_parts(); - Ok(Identity { der, pass }) - } + /// Constructs an identity from a PEM-formatted private key and certificate + /// chain, leaf certificate first. + /// + /// The key may be PKCS #8, PKCS #1 (RSA) or SEC1 (EC). Returns an error if + /// the key does not match the leaf certificate. + pub fn from_pem(key: &[u8], cert: &[u8]) -> Result { + let mut pem = Zeroizing::new(Vec::with_capacity(key.len() + cert.len() + 1)); + pem.extend_from_slice(key); + pem.push(b'\n'); + pem.extend_from_slice(cert); + + // Mirror `reqwest::Identity::from_pem`, which uses the last private + // key in the buffer. + let mut keys = PrivateKeyDer::pem_slice_iter(&pem).collect::, _>>()?; + let key = keys.pop().ok_or(TlsError::NoPrivateKey)?; + keys.iter_mut().for_each(Zeroize::zeroize); + let certs = CertificateDer::pem_slice_iter(&pem).collect::, _>>()?; + if certs.is_empty() { + return Err(TlsError::NoCertificate); + } - /// Wraps [`reqwest::Identity::from_pkcs12_der`]. - pub fn from_pkcs12_der(der: Vec, pass: String) -> Result { - let _ = reqwest::Identity::from_pkcs12_der(&der, &pass)?; - Ok(Identity { der, pass }) + // reqwest only checks that the key matches the certificate when the + // client is built, so check here to report the error up front. + let provider = rustls::crypto::aws_lc_rs::default_provider(); + rustls::sign::CertifiedKey::from_der(certs, key, &provider).map_err(TlsError::Identity)?; + let _ = reqwest::Identity::from_pem(&pem)?; + + Ok(Identity { + pem: std::mem::take(&mut *pem), + }) } } impl From for reqwest::Identity { fn from(id: Identity) -> Self { - reqwest::Identity::from_pkcs12_der(&id.der, &id.pass).expect("known to be a valid identity") + reqwest::Identity::from_pem(&id.pem).expect("known to be a valid identity") } } @@ -65,16 +112,27 @@ pub struct Certificate { } impl Certificate { - /// Wraps [`reqwest::Certificate::from_pem`]. - pub fn from_pem(pem: &[u8]) -> native_tls::Result { - Ok(Certificate { - der: native_tls::Certificate::from_pem(pem)?.to_der()?, - }) + /// Constructs a certificate from the first certificate in a PEM-formatted + /// buffer. + pub fn from_pem(pem: &[u8]) -> Result { + let der = CertificateDer::pem_slice_iter(pem) + .next() + .ok_or(TlsError::NoCertificate)??; + Self::from_der(&der) } - /// Wraps [`reqwest::Certificate::from_der`]. - pub fn from_der(der: &[u8]) -> native_tls::Result { - let _ = native_tls::Certificate::from_der(der)?; + /// Constructs a certificate from a DER-formatted buffer. + pub fn from_der(der: &[u8]) -> Result { + // Parse the certificate as a trust anchor, as the verifier does when + // the client is built. + rustls::RootCertStore::empty() + .add(CertificateDer::from_slice(der).into_owned()) + .map_err(|e| match e { + rustls::Error::InvalidCertificate(e) => TlsError::Certificate(e), + e => TlsError::Certificate(rustls::CertificateError::Other(rustls::OtherError( + Arc::new(e), + ))), + })?; Ok(Certificate { der: der.into() }) } } diff --git a/src/ccsr/tests/client.rs b/src/ccsr/tests/client.rs index 17798cc30dcbc..96ea1786f0145 100644 --- a/src/ccsr/tests/client.rs +++ b/src/ccsr/tests/client.rs @@ -445,8 +445,8 @@ fn test_invalid_tls_config_returns_error() { // Verify that invalid TLS material is caught at construction time, not at // ClientConfig::build() time (where it previously caused a panic via .unwrap()). - let err = Identity::from_pkcs12_der(vec![0xDE, 0xAD], "".into()); - assert!(err.is_err(), "garbage PKCS#12 should be rejected"); + let err = Identity::from_pem(b"not a key", b"not a certificate"); + assert!(err.is_err(), "garbage PEM identity should be rejected"); let err = Certificate::from_pem(b"not a certificate"); assert!(err.is_err(), "garbage PEM cert should be rejected"); @@ -455,6 +455,58 @@ fn test_invalid_tls_config_returns_error() { assert!(err.is_err(), "garbage DER cert should be rejected"); } +/// Returns a PEM-encoded PKCS #8 key and a self-signed certificate for it. +fn self_signed_pem() -> (Vec, Vec) { + use openssl::asn1::Asn1Time; + use openssl::ec::{EcGroup, EcKey}; + use openssl::hash::MessageDigest; + use openssl::nid::Nid; + use openssl::pkey::PKey; + use openssl::x509::{X509, X509NameBuilder}; + + let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).unwrap(); + let key = PKey::from_ec_key(EcKey::generate(&group).unwrap()).unwrap(); + let mut name = X509NameBuilder::new().unwrap(); + name.append_entry_by_text("CN", "ccsr-test").unwrap(); + let name = name.build(); + let mut cert = X509::builder().unwrap(); + cert.set_version(2).unwrap(); + cert.set_subject_name(&name).unwrap(); + cert.set_issuer_name(&name).unwrap(); + cert.set_pubkey(&key).unwrap(); + cert.set_not_before(&Asn1Time::days_from_now(0).unwrap()) + .unwrap(); + cert.set_not_after(&Asn1Time::days_from_now(1).unwrap()) + .unwrap(); + cert.sign(&key, MessageDigest::sha256()).unwrap(); + ( + key.private_key_to_pem_pkcs8().unwrap(), + cert.build().to_pem().unwrap(), + ) +} + +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `OPENSSL_init_ssl` on OS `linux` +fn test_pem_identity() { + use mz_ccsr::tls::Certificate; + + let (key, cert) = self_signed_pem(); + let ident = Identity::from_pem(&key, &cert).unwrap(); + assert_eq!(format!("{ident:?}"), "Identity { .. }"); + mz_ccsr::ClientConfig::new(reqwest::Url::parse("https://localhost").unwrap()) + .add_root_certificate(Certificate::from_pem(&cert).unwrap()) + .identity(ident) + .build() + .unwrap(); + + let (other_key, _) = self_signed_pem(); + let err = Identity::from_pem(&other_key, &cert).unwrap_err(); + assert!( + err.to_string().contains("KeyMismatch"), + "unexpected error: {err}" + ); +} + #[mz_ore::test] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_method` on OS `linux` fn test_stack_from_pem_error() { diff --git a/src/storage-types/Cargo.toml b/src/storage-types/Cargo.toml index d7e8bbcec13db..afd5a0c5021a4 100644 --- a/src/storage-types/Cargo.toml +++ b/src/storage-types/Cargo.toml @@ -58,8 +58,6 @@ mz-sql-server-util = { path = "../sql-server-util" } mz-timely-util = { path = "../timely-util" } mz-tls-util = { path = "../tls-util" } mz-tracing = { path = "../tracing" } -native-tls.workspace = true -openssl.workspace = true proptest = { workspace = true, optional = true } proptest-derive = { workspace = true, optional = true } prost.workspace = true diff --git a/src/storage-types/src/errors.rs b/src/storage-types/src/errors.rs index e3f0c4e4494ba..ec1e2b6578ca9 100644 --- a/src/storage-types/src/errors.rs +++ b/src/storage-types/src/errors.rs @@ -1202,9 +1202,7 @@ pub enum CsrConnectError { #[error("ssh: {0}")] Ssh(#[source] anyhow::Error), #[error(transparent)] - NativeTls(#[from] native_tls::Error), - #[error(transparent)] - Openssl(#[from] openssl::error::ErrorStack), + Tls(#[from] mz_ccsr::tls::TlsError), #[error(transparent)] Dns(#[from] mz_ore::netio::DnsResolutionError), #[error(transparent)] diff --git a/src/testdrive/Cargo.toml b/src/testdrive/Cargo.toml index 9061bc13cc13b..644d2d88d87a2 100644 --- a/src/testdrive/Cargo.toml +++ b/src/testdrive/Cargo.toml @@ -53,6 +53,7 @@ mz-sql-parser = { path = "../sql-parser" } mz-sql-server-util = { path = "../sql-server-util" } mz-storage-types = { path = "../storage-types" } mz-tls-util = { path = "../tls-util" } +openssl.workspace = true parquet.workspace = true postgres_array.workspace = true prost.workspace = true diff --git a/src/testdrive/src/action.rs b/src/testdrive/src/action.rs index dbfd0321dd09d..7df175bee15e6 100644 --- a/src/testdrive/src/action.rs +++ b/src/testdrive/src/action.rs @@ -35,6 +35,7 @@ use mz_ore::error::ErrorExt; use mz_ore::metrics::MetricsRegistry; use mz_ore::now::SYSTEM_TIME; use mz_ore::retry::Retry; +use mz_ore::secure::Zeroizing; use mz_ore::task; use mz_ore::url::SensitiveUrl; use mz_persist_client::cache::PersistClientCache; @@ -47,6 +48,7 @@ use mz_postgres_util::{ }; use mz_sql::catalog::EnvironmentId; use mz_tls_util::make_tls; +use openssl::pkcs12::Pkcs12; use rdkafka::ClientConfig; use rdkafka::producer::Producer; use regex::{Captures, Regex}; @@ -1181,9 +1183,9 @@ pub async fn create_state( let mut ccsr_config = mz_ccsr::ClientConfig::new(schema_registry_url.clone()); if let Some(cert_path) = &config.cert_path { - let cert = fs::read(cert_path).context("reading cert")?; - let pass = config.cert_password.as_deref().unwrap_or("").to_owned(); - let ident = mz_ccsr::tls::Identity::from_pkcs12_der(cert, pass) + let keystore = Zeroizing::new(fs::read(cert_path).context("reading cert")?); + let pass = config.cert_password.as_deref().unwrap_or(""); + let ident = identity_from_pkcs12_der(&keystore, pass) .context("reading keystore file as pkcs12")?; ccsr_config = ccsr_config.identity(ident); } @@ -1311,6 +1313,28 @@ pub async fn create_state( Ok((state, pgconn_task)) } +/// Converts a PKCS #12 keystore into a schema registry client identity. +/// +/// The same keystore is handed to librdkafka, which reads PKCS #12 natively, +/// while the schema registry client's rustls backend only accepts PEM. +fn identity_from_pkcs12_der(der: &[u8], pass: &str) -> anyhow::Result { + let parsed = Pkcs12::from_der(der)?.parse2(pass)?; + let key = Zeroizing::new( + parsed + .pkey + .context("keystore has no private key")? + .private_key_to_pem_pkcs8()?, + ); + let mut chain = parsed + .cert + .context("keystore has no certificate")? + .to_pem()?; + for ca in parsed.ca.into_iter().flatten() { + chain.extend(ca.to_pem()?); + } + Ok(mz_ccsr::tls::Identity::from_pem(&key, &chain)?) +} + async fn create_materialize_state( config: &&Config, materialize_catalog_config: Option, diff --git a/test/kafka-auth/test-schema-registry-mssl.td b/test/kafka-auth/test-schema-registry-mssl.td index 25210cc7d95ef..164a9fe3ec945 100644 --- a/test/kafka-auth/test-schema-registry-mssl.td +++ b/test/kafka-auth/test-schema-registry-mssl.td @@ -57,15 +57,14 @@ contains:error sending request for url ) contains:error sending request for url -# This is a bad error message to indicate "invalid client certificate" but -# it's not under our control. +# Client key does not match the client certificate. ! CREATE CONNECTION schema_registry_invalid TO CONFLUENT SCHEMA REGISTRY ( URL 'https://mssl.schema-registry.local:8082', SSL CERTIFICATE = '${schema-registry-crt}', SSL KEY = SECRET kafka_key, SSL CERTIFICATE AUTHORITY = '${ca-crt}' ) -contains:key values mismatch +contains:keys may not be consistent: KeyMismatch # ==> Test without an SSH tunnel. <== diff --git a/test/kafka-auth/test-schema-registry-ssl-basic.td b/test/kafka-auth/test-schema-registry-ssl-basic.td index 195f316334588..d39b1497e9714 100644 --- a/test/kafka-auth/test-schema-registry-ssl-basic.td +++ b/test/kafka-auth/test-schema-registry-ssl-basic.td @@ -40,7 +40,7 @@ $ kafka-ingest topic=avro-data format=avro schema=${schema} timestamp=1 ! CREATE CONNECTION schema_registry_invalid TO CONFLUENT SCHEMA REGISTRY ( URL 'https://ssl-basic.schema-registry.local:8082' ) -contains:certificate verify failed +contains:invalid peer certificate: UnknownIssuer ! CREATE CONNECTION schema_registry_invalid TO CONFLUENT SCHEMA REGISTRY ( URL 'https://ssl-basic.schema-registry.local:8082', @@ -166,20 +166,20 @@ contains:requires both SSL KEY and SSL CERTIFICATE contains:requires both SSL KEY and SSL CERTIFICATE ! ALTER CONNECTION schema_registry SET (SSL KEY = SECRET invalid_secret), SET (SSL CERTIFICATE = 'x') WITH (VALIDATE = true); -contains:No supported data to decode +contains:no private key found in PEM input ! ALTER CONNECTION schema_registry SET (SSL CERTIFICATE AUTHORITY = 'x') WITH (VALIDATE = true); -contains:CERTIFICATE +contains:no certificate found in PEM input > ALTER CONNECTION schema_registry RESET (SSL KEY); > ALTER CONNECTION schema_registry RESET (SSL CERTIFICATE); ! ALTER CONNECTION schema_registry RESET (SSL CERTIFICATE AUTHORITY) WITH (VALIDATE = true); -contains:self-signed certificate in certificate chain +contains:invalid peer certificate: UnknownIssuer ! ALTER CONNECTION schema_registry RESET (SSL KEY), RESET (SSL CERTIFICATE), RESET (SSL CERTIFICATE AUTHORITY); -contains:self-signed certificate in certificate chain +contains:invalid peer certificate: UnknownIssuer ! ALTER CONNECTION schema_registry RESET (USERNAME); contains:Unauthorized diff --git a/test/kafka-auth/test-schema-registry-ssl.td b/test/kafka-auth/test-schema-registry-ssl.td index 52c62d3ed88d8..de9cc4d54fc1f 100644 --- a/test/kafka-auth/test-schema-registry-ssl.td +++ b/test/kafka-auth/test-schema-registry-ssl.td @@ -37,7 +37,7 @@ contains:invalid HTTP version parsed ! CREATE CONNECTION schema_registry_invalid TO CONFLUENT SCHEMA REGISTRY ( URL 'https://ssl.schema-registry.local:8082' ) -contains:certificate verify failed +contains:invalid peer certificate: UnknownIssuer # ==> Test without an SSH tunnel. <==