From 07e1a96902c26988f58bd46e339851f94f997558 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 08:30:45 -0700 Subject: [PATCH 1/3] feat(ra-tls): add generate_self_signed_ra_cert --- dstack/ra-tls/src/cert.rs | 49 +++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/dstack/ra-tls/src/cert.rs b/dstack/ra-tls/src/cert.rs index ed3d71583..a08978304 100644 --- a/dstack/ra-tls/src/cert.rs +++ b/dstack/ra-tls/src/cert.rs @@ -560,13 +560,7 @@ pub fn generate_ra_cert_with_app_id( let ca = CaCert::new(ca_cert_pem, ca_key_pem)?; let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; - let pubkey = key.public_key_der(); - - let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); - - let attestation = Attestation::quote_with_app_id(&report_data, app_id) - .context("Failed to get quote for cert pubkey")? - .into_versioned(); + let attestation = quote_cert_pubkey(&key, app_id)?; // Build certificate request with all extensions let req = CertRequest::builder() @@ -581,6 +575,47 @@ pub fn generate_ra_cert_with_app_id( }) } +/// Generate a self-signed certificate with RA-TLS quote and event log. +/// +/// The quote's report data binds this certificate's public key, so a peer that +/// verifies RA-TLS certificates by attestation learns the sender's identity from +/// the quote rather than from the issuer. No CA — shared or otherwise — is needed +/// to mint it. +/// +/// Use this against servers that verify client certificates by attestation. The +/// certificate is rejected by servers that pin an issuer CA, which is why callers +/// that must interoperate with older peers keep using [`generate_ra_cert`]. +#[cfg(feature = "quote")] +pub fn generate_self_signed_ra_cert(app_id: Option<[u8; 20]>) -> Result { + use rcgen::{KeyPair, PKCS_ECDSA_P256_SHA256}; + + let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; + let attestation = quote_cert_pubkey(&key, app_id)?; + + let cert = CertRequest::builder() + .subject("RA-TLS Self-Signed Cert") + .key(&key) + .attestation(&attestation) + .usage_client_auth(true) + .build() + .self_signed() + .context("failed to self-sign certificate")?; + Ok(CertPair { + cert_pem: cert.pem(), + key_pem: key.serialize_pem(), + }) +} + +/// Take a quote over `key`'s public key, so the attestation binds the certificate +/// that will carry it. +#[cfg(feature = "quote")] +fn quote_cert_pubkey(key: &KeyPair, app_id: Option<[u8; 20]>) -> Result { + let report_data = QuoteContentType::RaTlsCert.to_report_data(&key.public_key_der()); + Ok(Attestation::quote_with_app_id(&report_data, app_id) + .context("Failed to get quote for cert pubkey")? + .into_versioned()) +} + #[cfg(test)] mod tests { use super::*; From b302c47007931ccdbfd7db1e7e6f8e3f8929dc3b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 08:30:45 -0700 Subject: [PATCH 2/3] refactor(guest): mint self-issued RA-TLS certs for KMS calls --- CHANGELOG.md | 1 + docs/encrypted-env-spec.md | 10 +++++--- dstack/cert-client/src/lib.rs | 15 +++++------- dstack/dstack-types/src/lib.rs | 4 --- dstack/dstack-util/src/main.rs | 34 +++++++++++--------------- dstack/dstack-util/src/system_setup.rs | 28 ++++++++++++--------- 6 files changed, 44 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc12db890..d998d7122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - sdk: the Go and Python compose-hash helpers silently dropped every app-compose field they did not declare, so `getComposeHash` returned a digest for an app-compose that was not the one being deployed — and that digest is what gets whitelisted on chain. The missing fields are named above; both now keep unrecognised keys as well, so a guest that gains a field before the SDK does still hashes correctly ### Changed +- dstack-util, cert-client: guests mint self-issued RA-TLS client certificates for KMS calls instead of fetching the temp CA and minting from it. The quote inside the certificate is the identity the KMS authenticates, so no CA material is needed; the KMS root CA now comes from `GetMeta`, over the same unauthenticated connection as before and pinned afterwards by `verify_key_provider_id` against the measured `app_compose.key_provider_id`. `.appkeys.json` no longer carries `tmp_ca_key`/`tmp_ca_cert`; readers ignore them in files written by older images. **This narrows KMS compatibility**: a guest built from this change is refused at the TLS handshake by any KMS that still pins the temp CA, which is every release up to and including 0.5.11, so it requires a KMS carrying the attestation-based client verifier. Guests built before this change are unaffected and keep working against both - kms: client certificates are authenticated by the attestation they carry rather than by their issuer. Rocket configures mutual TLS through rustls' `WebPkiClientVerifier`, which pins a CA — but an RA-TLS certificate is self-issued and carries its identity in a TEE quote, so there is nothing to chain to. `GetTempCaCert` bridged the gap by handing every caller a shared CA private key purely so the minted certificate would chain somewhere; the CA established nothing (its key is public by design, and the endpoint is unauthenticated) and the check that has always carried the meaning is the quote verification that runs afterwards. The KMS now hands rustls a verifier that requires an attestation and ignores the issuer. Nothing changes for callers: guests and KMS-to-KMS onboarding still mint their client certificates from the temp CA, and those are now accepted for the attestation they carry. What changes is that the TLS layer went from admitting any certificate signed by a public key to requiring an attested one, and that a self-issued certificate is now accepted — which is what lets callers be migrated off `GetTempCaCert` in a follow-up. `[rpc.tls.mutual]` is no longer the trust anchor and is dropped from `kms.toml` and the KMS config templates; leaving it in an existing deployment's config is inert. The gateway's `[tls.mutual]` is unaffected — it pins the KMS root CA, which is a real trust anchor - guest-agent: the `/metrics` exposition gains a conventions-compliant `dstack_guest_*` series set (application prefix; `_bytes`/`_seconds` unit suffixes; no `_total` suffix on gauges, which reads as a counter to every tool; the four OS/kernel/CPU gauges folded into one `dstack_guest_info`; `disk_used_ratio` 0–1 instead of a percentage). The old `system_*`/`disk_*` names are still emitted verbatim, marked deprecated, and will be removed in a future release — this endpoint is tenant-facing, so existing dashboards get a migration window - dstack-util: a CVM re-registers with the gateway node that last accepted it, before falling back to the configured order. The list used to be walked from the top every time, so every CVM piled onto the first URL and the whole fleet snapped back to it the moment it recovered from an outage — and each move rewrites the instance record from a different node's memory diff --git a/docs/encrypted-env-spec.md b/docs/encrypted-env-spec.md index c204a16e5..c56afa1ce 100644 --- a/docs/encrypted-env-spec.md +++ b/docs/encrypted-env-spec.md @@ -170,9 +170,7 @@ Path inside TEE: `/dstack/.host-shared/.appkeys.json` "key_provider": { "Kms": { "url": "https://kms.example.com/prpc", - "pubkey": "...", - "tmp_ca_key": "-----BEGIN PRIVATE KEY-----\n...", - "tmp_ca_cert": "-----BEGIN CERTIFICATE-----\n..." + "pubkey": "..." } } } @@ -200,11 +198,15 @@ Rust externally tagged enum — an object with exactly one key: {"None": {"key": ""}} {"Local": {"key": "", "mr": ""}} {"Tpm": {"key": "", "pubkey": ""}} -{"Kms": {"url": "...", "pubkey": "", "tmp_ca_key": "", "tmp_ca_cert": ""}} +{"Kms": {"url": "...", "pubkey": ""}} ``` The tag is one of `"None"` / `"Local"` / `"Tpm"` / `"Kms"`. +`Kms` carried `tmp_ca_key` / `tmp_ca_cert` before guests switched to self-issued +RA-TLS client certificates. Readers ignore the extra fields, so files written by +older guest images still parse. + ## Runtime File/Path Contract (dstack) For dstack runtime integration, treat these names/locations as protocol-level diff --git a/dstack/cert-client/src/lib.rs b/dstack/cert-client/src/lib.rs index e55689b52..e7880bb43 100644 --- a/dstack/cert-client/src/lib.rs +++ b/dstack/cert-client/src/lib.rs @@ -10,7 +10,7 @@ use dstack_types::{AppKeys, KeyProvider}; use ra_rpc::client::{RaClient, RaClientConfig}; use ra_tls::{ attestation::AttestationVerifier, - cert::{generate_ra_cert, CaCert, CertSigningRequestV2}, + cert::{generate_self_signed_ra_cert, CaCert, CertSigningRequestV2}, }; pub enum CertRequestClient { @@ -70,14 +70,11 @@ impl CertRequestClient { .context("Failed to create CA")?; Ok(CertRequestClient::Local { ca: Box::new(ca) }) } - KeyProvider::Kms { - url, - tmp_ca_key, - tmp_ca_cert, - .. - } => { - let client_cert = generate_ra_cert(tmp_ca_cert.clone(), tmp_ca_key.clone()) - .context("Failed to generate RA cert")?; + KeyProvider::Kms { url, .. } => { + // Self-issued: the KMS authenticates the quote inside this certificate, + // not whoever signed it. + let client_cert = + generate_self_signed_ra_cert(None).context("Failed to generate RA cert")?; let ra_client = RaClientConfig::builder() .remote_uri(url.clone()) .tls_client_cert(client_cert.cert_pem) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 448b350d7..f898a7fb4 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -2263,8 +2263,6 @@ pub enum KeyProvider { url: String, #[serde(with = "hex_bytes")] pubkey: Vec, - tmp_ca_key: String, - tmp_ca_cert: String, }, } @@ -2326,8 +2324,6 @@ mod key_provider_tests { let kms = KeyProvider::Kms { url: "https://kms.example".into(), pubkey: vec![0xab; 32], - tmp_ca_key: String::new(), - tmp_ca_cert: String::new(), }; assert_eq!(kms.id(), &[0xab; 32]); } diff --git a/dstack/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs index d4c56a93d..31e010376 100644 --- a/dstack/dstack-util/src/main.rs +++ b/dstack/dstack-util/src/main.rs @@ -14,7 +14,7 @@ use k256::schnorr::SigningKey; use ra_rpc::Attestation; use ra_tls::{ attestation::{AttestationQuote, QuoteContentType, VersionedAttestation}, - cert::{generate_ra_cert, generate_ra_cert_with_app_id}, + cert::{generate_ra_cert, generate_self_signed_ra_cert}, kdf::{derive_key, derive_p256_key_pair_from_bytes}, rcgen::KeyPair, }; @@ -633,13 +633,13 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { None }; - // Step 1: Get temporary CA certificate + // Step 1: Fetch the KMS root CA eprintln!("Connecting to KMS: {kms_url}"); let tls_no_check = root_ca_pem.is_none(); if tls_no_check { eprintln!("Warning: no --root-ca provided, TLS certificate verification is disabled for initial connection"); } - let tmp_ca = { + let root_ca = { let client = RaClientConfig::builder() .remote_uri(kms_url.clone()) .tls_no_check(tls_no_check) @@ -648,21 +648,17 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { .build() .into_client() .context("failed to create client")?; - let kms_client = KmsClient::new(client); - kms_client - .get_temp_ca_cert() + KmsClient::new(client) + .get_meta() .await - .context("Failed to get temp CA cert")? + .context("Failed to get KMS meta")? + .ca_cert }; - // Step 2: Generate RA-TLS client certificate + // Step 2: Generate a self-issued RA-TLS client certificate. The quote inside it + // is the identity the KMS authenticates, so no CA material is needed. let app_id = decode_app_id(args.app_id.as_deref())?; - let cert_pair = generate_ra_cert_with_app_id( - tmp_ca.temp_ca_cert.clone(), - tmp_ca.temp_ca_key.clone(), - app_id, - ) - .context("Failed to generate RA cert")?; + let cert_pair = generate_self_signed_ra_cert(app_id).context("Failed to generate RA cert")?; // Step 3: Create authenticated client and request app keys let ra_client = RaClientConfig::builder() @@ -671,7 +667,7 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { .remote_uri(kms_url.clone()) .tls_client_cert(cert_pair.cert_pem) .tls_client_key(cert_pair.key_pem) - .tls_ca_cert(tmp_ca.ca_cert.clone()) + .tls_ca_cert(root_ca.clone()) .build() .into_client() .context("Failed to create RA client")?; @@ -686,13 +682,13 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { .context("Failed to get app key")?; // Step 4: Build AppKeys structure - let (_, ca_pem) = x509_parser::pem::parse_x509_pem(tmp_ca.ca_cert.as_bytes()) - .context("Failed to parse CA cert")?; + let (_, ca_pem) = + x509_parser::pem::parse_x509_pem(root_ca.as_bytes()).context("Failed to parse CA cert")?; let x509 = ca_pem.parse_x509().context("Failed to parse CA cert")?; let root_pubkey = x509.public_key().raw.to_vec(); let keys = utils::AppKeys { - ca_cert: tmp_ca.ca_cert, + ca_cert: root_ca, disk_crypt_key: response.disk_crypt_key, env_crypt_key: response.env_crypt_key, k256_key: response.k256_key, @@ -701,8 +697,6 @@ async fn cmd_get_keys(args: GetKeysArgs) -> Result<()> { key_provider: KeyProvider::Kms { url: kms_url, pubkey: root_pubkey, - tmp_ca_key: tmp_ca.temp_ca_key, - tmp_ca_cert: tmp_ca.temp_ca_cert, }, }; diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 7dc2947bf..58cd97248 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -36,7 +36,7 @@ use ra_rpc::{ }; use ra_tls::{ attestation::{detect_tee_variant, AttestationVerifier, QuoteContentType, TeeVariant}, - cert::{generate_ra_cert, CertConfigV2, CertSigningRequestV2, Csr}, + cert::{generate_self_signed_ra_cert, CertConfigV2, CertSigningRequestV2, Csr}, }; use rand::Rng as _; use safe_write::{safe_write, safe_write_with_mode}; @@ -2505,16 +2505,24 @@ impl<'a> Stage0<'a> { async fn request_app_keys_from_kms_url(&self, kms_url: String) -> Result { info!("Requesting app keys from KMS: {kms_url}"); - let tmp_ca = { - info!("Getting temp ca cert"); + // Learn the KMS root CA over an unauthenticated connection. It is pinned + // immediately afterwards by `verify_key_provider_id`, which compares it + // against the measured `app_compose.key_provider_id`, so a CA substituted + // here does not survive to the point where the keys get used. + let root_ca = { + info!("Getting KMS root CA"); let client = RaClient::new(kms_url.clone(), true)?; let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(client); kms_client - .get_temp_ca_cert() + .get_meta() .await - .context("Failed to get temp ca cert")? + .context("Failed to get KMS meta")? + .ca_cert }; - let cert_pair = generate_ra_cert(tmp_ca.temp_ca_cert.clone(), tmp_ca.temp_ca_key.clone())?; + // Self-issued: the quote inside the certificate is the identity, so there is + // no CA to fetch and no shared private key to hold. + let cert_pair = + generate_self_signed_ra_cert(None).context("Failed to generate RA-TLS cert")?; let attestation_verifier = attestation_verifier(&self.shared.sys_config)?; let ra_client = RaClientConfig::builder() .tls_no_check(false) @@ -2522,7 +2530,7 @@ impl<'a> Stage0<'a> { .remote_uri(kms_url.clone()) .tls_client_cert(cert_pair.cert_pem) .tls_client_key(cert_pair.key_pem) - .tls_ca_cert(tmp_ca.ca_cert.clone()) + .tls_ca_cert(root_ca.clone()) .attestation_verifier(attestation_verifier) .cert_validator(Box::new(validate_kms_rpc_cert)) .build() @@ -2540,13 +2548,13 @@ impl<'a> Stage0<'a> { emit_runtime_event("os-image-hash", &response.os_image_hash) .context("failed to extend os-image-hash to the launch measurement")?; - let (_, ca_pem) = x509_parser::pem::parse_x509_pem(tmp_ca.ca_cert.as_bytes()) + let (_, ca_pem) = x509_parser::pem::parse_x509_pem(root_ca.as_bytes()) .context("Failed to parse ca cert")?; let x509 = ca_pem.parse_x509().context("Failed to parse ca cert")?; let root_pubkey = x509.public_key().raw.to_vec(); let keys = AppKeys { - ca_cert: tmp_ca.ca_cert, + ca_cert: root_ca, disk_crypt_key: response.disk_crypt_key, env_crypt_key: response.env_crypt_key, k256_key: response.k256_key, @@ -2555,8 +2563,6 @@ impl<'a> Stage0<'a> { key_provider: KeyProvider::Kms { url: kms_url, pubkey: root_pubkey, - tmp_ca_key: tmp_ca.temp_ca_key, - tmp_ca_cert: tmp_ca.temp_ca_cert, }, }; Ok(keys) From 3d5f600e87d9bfc1cacc81551ddcc1716a6e40b3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 09:34:03 -0700 Subject: [PATCH 3/3] refactor(kms): mint self-issued RA-TLS certs for onboarding --- CHANGELOG.md | 2 +- docs/deployment.md | 14 +++-- docs/security/public-security-reports.md | 2 +- docs/security/security-best-practices.md | 2 +- docs/security/security-model.md | 2 +- dstack/kms/rpc/proto/kms_rpc.proto | 16 +++--- dstack/kms/src/main_service.rs | 20 ++++--- dstack/kms/src/onboard_service.rs | 61 +++++++++++++--------- dstack/ra-rpc/src/client.rs | 18 ------- dstack/ra-rpc/src/ratls_client_verifier.rs | 11 ++-- dstack/ra-rpc/tests/ratls_client_auth.rs | 15 +++--- 11 files changed, 76 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d998d7122..6ec7ac885 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - sdk: the Go and Python compose-hash helpers silently dropped every app-compose field they did not declare, so `getComposeHash` returned a digest for an app-compose that was not the one being deployed — and that digest is what gets whitelisted on chain. The missing fields are named above; both now keep unrecognised keys as well, so a guest that gains a field before the SDK does still hashes correctly ### Changed -- dstack-util, cert-client: guests mint self-issued RA-TLS client certificates for KMS calls instead of fetching the temp CA and minting from it. The quote inside the certificate is the identity the KMS authenticates, so no CA material is needed; the KMS root CA now comes from `GetMeta`, over the same unauthenticated connection as before and pinned afterwards by `verify_key_provider_id` against the measured `app_compose.key_provider_id`. `.appkeys.json` no longer carries `tmp_ca_key`/`tmp_ca_cert`; readers ignore them in files written by older images. **This narrows KMS compatibility**: a guest built from this change is refused at the TLS handshake by any KMS that still pins the temp CA, which is every release up to and including 0.5.11, so it requires a KMS carrying the attestation-based client verifier. Guests built before this change are unaffected and keep working against both +- dstack-util, cert-client, kms: guests and KMS-to-KMS onboarding mint self-issued RA-TLS client certificates instead of fetching the temp CA and minting from it. The quote inside the certificate is the identity the KMS authenticates, so no CA material is needed; the KMS root CA now comes from `GetMeta`, over the same unauthenticated connection as before and pinned afterwards by `verify_key_provider_id` against the measured `app_compose.key_provider_id`. `.appkeys.json` no longer carries `tmp_ca_key`/`tmp_ca_cert`; readers ignore them in files written by older images. Onboarding does the same and drops the `GetTempCaCert` round trip entirely: with no CA to fetch first, it now runs over a single client that carries both the client certificate and the `cert_validator`, so the source KMS's attestation is verified on the handshake that actually carries the root key rather than on an earlier, separate connection - `RaClient::new_mtls`, its only user, set no validator at all and is removed. The source is re-checked against local policy (`ensure_kms_allowed`) after the fetch instead of before it. `GetTempCaCert` now has no caller in this tree and is retained only so guest images built before the switch keep booting. **This narrows which peers work**: a guest or a joining KMS built from this change is refused at the TLS handshake by any KMS that still pins the temp CA, which is every release up to and including 0.5.11. So a guest needs a KMS at 0.6.0 or later, and onboarding needs a *source* at 0.6.0 or later - upgrading a 0.5.x cluster past 0.6.0 goes through 0.6.0 as a bridge hop, the same two-hop shape 0.5.4 already needs. Guests and KMS nodes built before this change are unaffected and keep working against both - kms: client certificates are authenticated by the attestation they carry rather than by their issuer. Rocket configures mutual TLS through rustls' `WebPkiClientVerifier`, which pins a CA — but an RA-TLS certificate is self-issued and carries its identity in a TEE quote, so there is nothing to chain to. `GetTempCaCert` bridged the gap by handing every caller a shared CA private key purely so the minted certificate would chain somewhere; the CA established nothing (its key is public by design, and the endpoint is unauthenticated) and the check that has always carried the meaning is the quote verification that runs afterwards. The KMS now hands rustls a verifier that requires an attestation and ignores the issuer. Nothing changes for callers: guests and KMS-to-KMS onboarding still mint their client certificates from the temp CA, and those are now accepted for the attestation they carry. What changes is that the TLS layer went from admitting any certificate signed by a public key to requiring an attested one, and that a self-issued certificate is now accepted — which is what lets callers be migrated off `GetTempCaCert` in a follow-up. `[rpc.tls.mutual]` is no longer the trust anchor and is dropped from `kms.toml` and the KMS config templates; leaving it in an existing deployment's config is inert. The gateway's `[tls.mutual]` is unaffected — it pins the KMS root CA, which is a real trust anchor - guest-agent: the `/metrics` exposition gains a conventions-compliant `dstack_guest_*` series set (application prefix; `_bytes`/`_seconds` unit suffixes; no `_total` suffix on gauges, which reads as a counter to every tool; the four OS/kernel/CPU gauges folded into one `dstack_guest_info`; `disk_used_ratio` 0–1 instead of a percentage). The old `system_*`/`disk_*` names are still emitted verbatim, marked deprecated, and will be removed in a future release — this endpoint is tenant-facing, so existing dashboards get a migration window - dstack-util: a CVM re-registers with the gateway node that last accepted it, before falling back to the configured order. The list used to be walked from the top every time, so every CVM piled onto the first URL and the whole fleet snapped back to it the moment it recovered from an outage — and each move rewrites the instance record from a different node's memory diff --git a/docs/deployment.md b/docs/deployment.md index dec73fe4b..7504cd3ff 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -430,13 +430,17 @@ Additional KMS instances can onboard from an existing KMS to share the same root **How it works:** 1. New KMS starts in onboard mode (empty `auto_bootstrap_domain`) -2. New KMS calls `GetTempCaCert` on source KMS -3. New KMS generates RA-TLS certificate with TDX quote -4. New KMS calls `GetKmsKey` with mTLS authentication -5. Source KMS verifies attestation via `bootAuth/kms` webhook -6. If approved, source KMS returns root keys +2. New KMS generates a self-issued RA-TLS certificate with a TDX quote +3. New KMS calls `GetKmsKey` with mTLS authentication +4. Source KMS verifies attestation via `bootAuth/kms` webhook +5. If approved, source KMS returns root keys +6. New KMS checks the source against its own policy before adopting the keys 7. Both KMS instances now derive identical keys +The source KMS must be running 0.6.0 or later, because it has to accept a self-issued +client certificate. Onboarding from an older source is not supported; upgrade that source +to 0.6.0 first. + **Configure new KMS for onboarding:** ```toml diff --git a/docs/security/public-security-reports.md b/docs/security/public-security-reports.md index d04b40628..89d5b0876 100644 --- a/docs/security/public-security-reports.md +++ b/docs/security/public-security-reports.md @@ -59,7 +59,7 @@ These issues were filed as concrete vulnerability reports, security audit findin | [#616](https://github.com/Dstack-TEE/dstack/issues/616) Host-controlled Docker registry mirror enables image substitution attacks | Closed | Not a production vulnerability | Registry mirrors are untrusted transport. Digest-pinned image references and measured compose configuration protect against substitution. No code fix was applied | | [#617](https://github.com/Dstack-TEE/dstack/issues/617) Guest agent exposes raw private keys to all local processes | Closed | Not a production vulnerability | dstack treats a CVM as one application trust domain. It does not provide per-container key isolation inside the same measured application. No code fix was applied | | [#618](https://github.com/Dstack-TEE/dstack/issues/618) Disk encryption disableable via kernel cmdline, not measured in RTMR | Closed | Not a production vulnerability | The kernel command line is measured into RTMR2, so changing `dstack.storage_encrypted=false` changes attestation evidence. No code fix was applied | -| [#619](https://github.com/Dstack-TEE/dstack/issues/619) KMS `get_temp_ca_cert` returns temp CA private key without authentication | Closed | Duplicate | The report duplicates the private advisory response for the temp CA bootstrap flow. [#1106](https://github.com/Dstack-TEE/dstack/pull/1106) removes the reason the temp CA existed - the KMS no longer pins it - so callers can be migrated off it; the RPC itself is retained until guests and KMS-to-KMS onboarding are | +| [#619](https://github.com/Dstack-TEE/dstack/issues/619) KMS `get_temp_ca_cert` returns temp CA private key without authentication | Closed | Duplicate | The report duplicates the private advisory response for the temp CA bootstrap flow. [#1106](https://github.com/Dstack-TEE/dstack/pull/1106) removes the reason the temp CA existed - the KMS no longer pins it - and guests and KMS-to-KMS onboarding now mint self-issued certificates, so the RPC has no caller left in-tree; it is retained only so guest images built before that switch keep booting | ## Related security roadmap and hardening diff --git a/docs/security/security-best-practices.md b/docs/security/security-best-practices.md index 91b3c0c2e..fbcd7c7f6 100644 --- a/docs/security/security-best-practices.md +++ b/docs/security/security-best-practices.md @@ -111,7 +111,7 @@ Development settings are intentionally easy to audit, but they are not productio - The KMS contract pins a concrete gateway app id. Do not use `gateway_app_id = "any"` for production traffic. - TEE quotes are evaluated by deployment policy, including TCB status and expected OS/application measurements. -The KMS TLS listener verifies client certificates by the attestation they carry rather than by an issuer CA, so it needs no `rpc.tls.mutual` section. It still accepts connections without a client certificate, because bootstrap and public metadata endpoints must be reachable before a client has an RA-TLS certificate. `GetTempCaCert` remains in use by guests and by KMS-to-KMS onboarding, which still mint their client certificates from that CA; it returns temp CA private material, so treat it as bootstrap-sensitive. +The KMS TLS listener verifies client certificates by the attestation they carry rather than by an issuer CA, so it needs no `rpc.tls.mutual` section. It still accepts connections without a client certificate, because bootstrap and public metadata endpoints must be reachable before a client has an RA-TLS certificate. `GetTempCaCert` has no caller left in-tree - guests and KMS-to-KMS onboarding both mint self-issued certificates - but is retained so guest images built before that switch keep booting; it returns temp CA private material, so treat it as bootstrap-sensitive. App key release and KMS key handover still require verified caller attestation from the RA-TLS client certificate. Certificate signing verifies the CSR signature and embedded attestation before signing. diff --git a/docs/security/security-model.md b/docs/security/security-model.md index 72791944e..6ff699544 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -341,7 +341,7 @@ The KMS Rocket TLS listener permits connections without a client certificate bec App key release and KMS key handover require verified caller attestation from the RA-TLS client certificate. Certificate signing verifies the CSR signature and the attestation embedded in the CSR before signing. -The unauthenticated or non-client-certificate surface includes bootstrap and temp-CA bootstrap material retrieval, env-encryption public-key retrieval, metadata, health, and metrics behavior documented for operators. `GetTempCaCert` returns temp CA private material and remains in use by guests and by KMS-to-KMS onboarding, which mint their client certificates from that CA; operators must treat it as bootstrap-sensitive rather than harmless public metadata. +The unauthenticated or non-client-certificate surface includes bootstrap and temp-CA bootstrap material retrieval, env-encryption public-key retrieval, metadata, health, and metrics behavior documented for operators. `GetTempCaCert` returns temp CA private material. It has no caller left in-tree - guests and KMS-to-KMS onboarding both mint self-issued certificates - but is retained so guest images built before that switch keep booting; operators must treat it as bootstrap-sensitive rather than harmless public metadata. ## Limitations diff --git a/dstack/kms/rpc/proto/kms_rpc.proto b/dstack/kms/rpc/proto/kms_rpc.proto index e491d9335..6d813d35f 100644 --- a/dstack/kms/rpc/proto/kms_rpc.proto +++ b/dstack/kms/rpc/proto/kms_rpc.proto @@ -108,16 +108,14 @@ service KMS { rpc GetMeta(google.protobuf.Empty) returns (GetMetaResponse); // Request the temporary CA certificate and key. // - // Both current callers - guests at boot, and KMS-to-KMS onboarding - fetch this CA - // and mint their client certificate from it, because the KMS used to pin the CA for - // mutual TLS and a self-issued certificate had nothing to chain to. + // Deprecated, and no longer called from this tree: guests and KMS-to-KMS onboarding + // both mint self-issued RA-TLS certificates now. It exists because the KMS used to + // pin this CA for mutual TLS, which left a self-issued certificate with nothing to + // chain to; the KMS verifies client certificates by the attestation they carry and + // ignores the issuer, so that CA no longer establishes anything. // - // That pin is gone: the KMS now verifies client certificates by the attestation - // they carry and ignores the issuer, so a self-issued certificate would be accepted - // on the same terms. Neither caller has been migrated yet, so this RPC still has to - // work; it is scheduled for removal once both are. - // - // Do not build new callers on it. A caller that only needs the KMS root CA can read + // Retained so guest images built before the switch keep booting. Remove once no + // supported guest image calls it. A caller that only needs the KMS root CA can read // `ca_cert` from `GetMeta` instead. rpc GetTempCaCert(google.protobuf.Empty) returns (GetTempCaCertResponse); // Sign a certificate diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 726a0a341..0554ec46c 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -496,18 +496,16 @@ impl KmsRpc for RpcHandler { /// Serve the temp CA certificate and key. /// - /// Both current callers - guests at boot, and KMS-to-KMS onboarding - /// ([`crate::onboard_service`]) - fetch this CA and mint their client certificate - /// from it, because the KMS used to pin the CA for mutual TLS and a self-issued - /// certificate had nothing to chain to. + /// Deprecated, and no longer called from this tree: guests and KMS-to-KMS + /// onboarding ([`crate::onboard_service`]) both mint self-issued RA-TLS + /// certificates now. It exists because the KMS used to pin this CA for mutual TLS, + /// which left a self-issued certificate with nothing to chain to; client + /// certificates are verified by the attestation they carry + /// (`ra_rpc::ratls_client_verifier`), so that CA no longer establishes anything. /// - /// That pin is gone: client certificates are now verified by the attestation they - /// carry (`ra_rpc::ratls_client_verifier`), so a self-issued certificate would be - /// accepted on the same terms. Neither caller has been migrated yet, so this RPC - /// still has to work. - /// - /// The key it returns authenticates nobody: it is handed to any caller. Removing - /// this RPC needs both callers migrated first. + /// The key it returns authenticates nobody: it is handed to any caller. Retained + /// so guest images built before the switch keep booting; remove once no supported + /// guest image calls it. async fn get_temp_ca_cert(self) -> Result { let self_boot_info = self .ensure_self_allowed() diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index 2d2d8d0af..c80687fb4 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -17,7 +17,7 @@ use dstack_kms_rpc::{ use fs_err as fs; use k256::ecdsa::SigningKey; use ra_rpc::{ - client::{CertInfo, RaClient, RaClientConfig}, + client::{CertInfo, RaClientConfig}, CallContext, RpcCall, }; use ra_tls::{ @@ -25,7 +25,7 @@ use ra_tls::{ AttestationVerifier, GetDeviceId, PlatformEvidence, QuoteContentType, VerifiedAttestation, VersionedAttestation, }, - cert::{CaCert, CertRequest}, + cert::CertRequest, rcgen::{Certificate, KeyPair, PKCS_ECDSA_P256_SHA256}, }; use safe_write::{safe_write, safe_write_with_mode}; @@ -542,9 +542,20 @@ impl Keys { ) -> Result { let attestation_slot = Arc::new(Mutex::new(None::)); let attestation_slot_out = attestation_slot.clone(); + // Self-issued: the source KMS authenticates the quote inside this certificate, + // not whoever signed it, so there is no CA to fetch first. This requires a + // source running 0.6.0 or later; older releases pin their temp CA and refuse a + // self-issued certificate at the handshake. + let (ra_cert, ra_key) = gen_ra_cert().await?; + // One client, not two. The connection that carries the root key is the one + // whose `cert_validator` runs, so the source's attestation is verified on the + // handshake that matters rather than on an earlier, separate connection. let client = RaClientConfig::builder() .tls_no_check(true) + .tls_built_in_root_certs(false) .remote_uri(other_kms_url.to_string()) + .tls_client_cert(ra_cert) + .tls_client_key(ra_key) .cert_validator(Box::new(move |info: Option| { let Some(info) = info else { bail!("Source KMS did not present a TLS certificate"); @@ -561,18 +572,15 @@ impl Keys { .attestation_verifier(attestation_verifier.clone()) .build() .into_client()?; - let mut kms_client = KmsClient::new(client); - - let tmp_ca = kms_client.get_temp_ca_cert().await?; - let (ra_cert, ra_key) = gen_ra_cert(tmp_ca.temp_ca_cert, tmp_ca.temp_ca_key).await?; - let ra_client = RaClient::new_mtls( - other_kms_url.into(), - ra_cert, - ra_key, - attestation_verifier.clone(), - ) - .context("Failed to create client")?; - kms_client = KmsClient::new(ra_client); + let kms_client = KmsClient::new(client); + + let info = dstack_client().info().await.context("Failed to get info")?; + let keys_res = kms_client + .get_kms_key(GetKmsKeyRequest { + vm_config: info.vm_config, + }) + .await?; + let source_attestation = attestation_slot .lock() .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))? @@ -582,12 +590,6 @@ impl Keys { .await .context("Source KMS is not allowed for onboarding")?; - let info = dstack_client().info().await.context("Failed to get info")?; - let keys_res = kms_client - .get_kms_key(GetKmsKeyRequest { - vm_config: info.vm_config, - }) - .await?; if keys_res.keys.len() != 1 { return Err(anyhow::anyhow!("Invalid keys")); } @@ -744,11 +746,16 @@ fn keccak256(msg: &[u8]) -> [u8; 32] { hasher.finalize().into() } -async fn gen_ra_cert(ca_cert_pem: String, ca_key_pem: String) -> Result<(String, String)> { +/// Mint the self-issued RA-TLS certificate this KMS presents while onboarding. +/// +/// The quote binds the certificate's own public key, which is the identity the source +/// KMS authenticates, so no CA is involved. The quote comes from the guest agent +/// (`app_attest`) rather than from `ra_tls`'s direct quote path, because the KMS is an +/// application inside a CVM and its attestation has to carry the agent's app info. +async fn gen_ra_cert() -> Result<(String, String)> { use ra_tls::cert::CertRequest; use ra_tls::rcgen::{KeyPair, PKCS_ECDSA_P256_SHA256}; - let ca = CaCert::new(ca_cert_pem, ca_key_pem)?; let key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; let pubkey = key.public_key_der(); let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); @@ -757,11 +764,13 @@ async fn gen_ra_cert(ca_cert_pem: String, ca_key_pem: String) -> Result<(String, .context("Failed to get quote")?; let attestation = VersionedAttestation::from_bytes(&response.attestation).context("Invalid attestation")?; - let req = CertRequest::builder() - .subject("RA-TLS TEMP Cert") + let cert = CertRequest::builder() + .subject("RA-TLS Self-Signed Cert") .attestation(&attestation) .key(&key) - .build(); - let cert = ca.sign(req).context("Failed to sign certificate")?; + .usage_client_auth(true) + .build() + .self_signed() + .context("Failed to self-sign certificate")?; Ok((cert.pem(), key.serialize_pem())) } diff --git a/dstack/ra-rpc/src/client.rs b/dstack/ra-rpc/src/client.rs index 38b2124eb..459ac42b0 100644 --- a/dstack/ra-rpc/src/client.rs +++ b/dstack/ra-rpc/src/client.rs @@ -110,24 +110,6 @@ impl RaClient { .context("failed to create client") } - pub fn new_mtls( - remote_uri: String, - cert_pem: String, - key_pem: String, - attestation_verifier: Arc, - ) -> Result { - RaClientConfig::builder() - .tls_no_check(true) - .tls_built_in_root_certs(false) - .remote_uri(remote_uri) - .tls_client_cert(cert_pem) - .tls_client_key(key_pem) - .attestation_verifier(attestation_verifier) - .build() - .into_client() - .context("failed to create client") - } - async fn try_validate_attestation(&self, response: &Response) -> Result<()> { let Some(validator) = &self.cert_validator else { return Ok(()); diff --git a/dstack/ra-rpc/src/ratls_client_verifier.rs b/dstack/ra-rpc/src/ratls_client_verifier.rs index d3c467808..b2cdaa7c6 100644 --- a/dstack/ra-rpc/src/ratls_client_verifier.rs +++ b/dstack/ra-rpc/src/ratls_client_verifier.rs @@ -13,9 +13,8 @@ //! //! [`RaTlsClientVerifier`] replaces the chain check with the check that actually //! carries meaning: the certificate must carry an attestation. Certificates minted -//! from that temp CA — which is what guests and KMS-to-KMS onboarding still send — -//! are accepted for that attestation rather than for their issuer, so nothing has to -//! change on the client side for them to keep working. Verifying that +//! from that temp CA are still accepted, on the same terms as self-issued ones, which +//! is what keeps guest images built before the switch working. Verifying that //! attestation — and deciding whether the app behind it is authorized — needs //! network I/O (collateral fetch, auth API) and stays where it already is, in //! [`crate::rocket_helper`] and the service handlers. Keeping the expensive half @@ -299,9 +298,9 @@ mod tests { #[test] fn accepts_ca_signed_cert() { - // What guests and KMS-to-KMS onboarding send today: a certificate minted from - // the KMS temp CA. That chain is no longer pinned, so it is accepted for the - // attestation it carries instead. + // Guest images built before the switch still mint their client certificate from + // the KMS temp CA. That chain is no longer pinned, so the certificate is + // accepted for the attestation it carries instead. let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256).unwrap(); let ca_cert = CertRequest::builder() .subject("Dstack Client Temp CA") diff --git a/dstack/ra-rpc/tests/ratls_client_auth.rs b/dstack/ra-rpc/tests/ratls_client_auth.rs index 9719c508c..bc3580e8e 100644 --- a/dstack/ra-rpc/tests/ratls_client_auth.rs +++ b/dstack/ra-rpc/tests/ratls_client_auth.rs @@ -6,10 +6,10 @@ //! clients by the attestation in their certificate rather than by its issuer. //! //! The cases that matter: -//! * a certificate minted from a temp CA is accepted (what guests and KMS-to-KMS -//! onboarding send today, and what used to be the only accepted shape); -//! * a self-issued RA-TLS certificate is accepted too, which is what lets clients -//! stop fetching CA material; +//! * a self-issued RA-TLS certificate is accepted (what guests and KMS-to-KMS +//! onboarding send); +//! * a certificate minted from a temp CA is also accepted, which is what keeps guest +//! images built before the switch working; //! * a certificate with no attestation is rejected during the handshake; //! * an anonymous connection still reaches the handler, so the unauthenticated //! RPCs stay reachable. @@ -159,16 +159,15 @@ async fn client_certs_are_authenticated_by_attestation_not_issuer() { } assert!(up, "server never came up on port {port}"); - // A self-issued RA-TLS cert is accepted and reaches the handler. No client sends - // one yet; this is the shape the CA pin used to reject. + // A self-issued RA-TLS cert is accepted and reaches the handler. let self_signed = client_cert(true, None); assert_eq!( probe(port, Some(&self_signed)).await.unwrap(), "cn=test client" ); - // A cert minted from the temp CA — what guests and KMS-to-KMS onboarding present - // today — keeps working. + // A cert minted from the temp CA — what guest images built before the switch + // present — keeps working. let (ca_cert, ca_key) = temp_ca(); let ca_signed = client_cert(true, Some((&ca_cert, &ca_key))); assert_eq!(