From 7f7748d7eaa1b7c634a4ab1833dd5edd1c199883 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 02:50:18 -0700 Subject: [PATCH 1/7] feat(guest-agent): add the dstack.guest.v1 proto surface --- .../guest-agent/rpc/proto/agent_rpc_v1.proto | 429 ++++++++++++++++++ dstack/guest-agent/rpc/src/generated.rs | 11 + 2 files changed, 440 insertions(+) create mode 100644 dstack/guest-agent/rpc/proto/agent_rpc_v1.proto diff --git a/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto b/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto new file mode 100644 index 000000000..86505a786 --- /dev/null +++ b/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto @@ -0,0 +1,429 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package dstack.guest.v1; + +// The versioned dstack guest agent API. +// +// Two services, one per trust surface, mirroring the two unversioned ones: +// +// `DstackGuestV1` internal socket (`/var/run/dstack.sock`), at `/v1` +// `WorkerV1` external listener, at `/prpc/v1` +// +// Each unversioned surface stays mounted where it was and is closed at exactly +// v0.5.11. The version a caller gets is decided by the URL path alone, never by +// a header. +// +// The two are not the same surface with different mounts. The internal socket +// is reachable only by the application itself, so `DstackGuestV1` hands out key +// material; the external listener is reachable by anyone who can route to the +// CVM, so `WorkerV1` never does. +// +// Conventions, applied without exception: +// - every request message is `Request`, every response +// `Response`, including for methods that take no arguments -- an +// empty message can gain a field, `google.protobuf.Empty` cannot; +// - a field's encoding is stated in its doc comment, not in its name. The +// fields that carry JSON documents say so and name who owns the schema; +// - a key is named by `(domain, algorithm)` and nothing else. There is no +// `purpose` field anywhere in v1. Encode roles in the domain string. +// +// v1 keys are NOT the v0 keys. The v1 KDF binds the algorithm and a versioned +// context tag alongside the domain, so the same name yields different key +// material here than on the unversioned surface, and secp256k1 and ed25519 no +// longer share one 32-byte secret. See `docs/guest-api-v1.md` for the +// byte-level construction. +// +// v1 serves only what genuinely needs the TEE: deriving keys from the app root +// key, and attesting. It is not an HSM and does not pretend to be one. +// +// So there is no `Sign` and no `Verify`. Any caller that can reach this socket +// can ask `GetKey` for the private key itself, so a server-side `Sign` grants +// no capability the caller does not already have -- it is pure computation +// behind an IPC round trip, and one more entry point to audit. Verifying is +// the same argument without even the key: the agent's answer arrives +// unattested, so a relying party gains nothing over checking it itself. +// Applications sign locally with a standard library, using the key `GetKey` +// returns; `docs/guest-api-v1.md` specifies chain verification normatively. +// +// There is no `EmitEvent` either: runtime RTMR3 events are system-owned as of +// 0.6.0 and cannot be extended by an app. +// +// The unversioned surface keeps `Sign` and `Verify` for v0.5.x clients. They +// are frozen there and are not carried forward. +service DstackGuestV1 { + // Issue a certificate for this application. + // + // The agent builds a CSR, signs it with the certificate's own key, and + // relays it to the KMS `SignCert` RPC (or to the local CA when the app runs + // without a KMS), then returns the chain the signer produced. Certificate + // issuance is the operation; v0 called this `GetTlsKey`, which named the + // by-product instead of the request. + // + // The private key is freshly generated for every call and is NOT derived + // from the app identity: none of the request fields feed it, and calling + // twice with the same arguments yields two unrelated keys. `GetKey` is the + // method that derives a stable, attestable key. + // + // This first cut serves only the integrated one-step mode, where the agent + // holds the key. A mode that signs a caller-supplied CSR or public key -- + // so the private key never leaves the caller -- is a plausible extension and + // would arrive as added fields or a sibling method, not as a change to what + // these fields mean. + rpc IssueCert(IssueCertRequest) returns (IssueCertResponse) {} + + // Derive an application key from `(domain, algorithm)` and return the private + // key with its signature chain. + rpc GetKey(GetKeyRequest) returns (GetKeyResponse) {} + + // Produce a versioned attestation over the given report data. + // + // The sole CVM attestation entry point in v1. The dstack-defined attestation + // format covers every supported platform and already carries the TDX quote + // and the event log, so v0's TDX-only `GetQuote` has nothing left to add: + // it answers on Intel TDX and nowhere else, and on GCP Confidential VMs it + // returns the TDX quote without the vTPM quote GCP's verification also + // binds. `docs/guest-api-v1.md` says how to extract a raw quote and event + // log from the attestation. `GetQuote` stays on the unversioned surface for + // v0.5.x clients. + rpc Attest(AttestRequest) returns (AttestResponse) {} + + // Collect GPU attestation evidence now, against a nonce the caller chooses. + // + // This answers "is the device I can talk to right now a genuine, CC-enabled + // NVIDIA GPU that signs my challenge", which `GpuInfo` cannot: that returns + // a record written at boot. Use it after anything that may have + // reinitialised the GPU -- a driver reload leaves a device that responds to + // NVML but can no longer attest -- and before submitting work you care + // about. + // + // Returns vendor-native evidence, not a local verdict, so a relying party + // can appraise it with its own verifier. Evidence still does not bind the + // GPU to this TD; see `AttestGpuResponse.bundles`. + rpc AttestGpu(AttestGpuRequest) returns (AttestGpuResponse) {} + + // Return this application's identity and measurements. + rpc Info(InfoRequest) returns (InfoResponse) {} + + // Return the guest agent version. + rpc Version(VersionRequest) returns (VersionResponse) {} +} + +// --------------------------------------------------------------------------- +// Certificate issuance +// --------------------------------------------------------------------------- + +message IssueCertRequest { + // Subject of the certificate to request. + string subject = 1; + // DNS alternative names for the certificate. + repeated string alt_names = 2; + // Include the attestation quote in the certificate (RA-TLS). + bool usage_ra_tls = 3; + // Key usage server auth. + bool usage_server_auth = 4; + // Key usage client auth. + bool usage_client_auth = 5; + // Certificate validity start, seconds since the UNIX epoch. Rejected when + // it is not earlier than `not_after`. + optional uint64 not_before = 6; + // Certificate validity end, seconds since the UNIX epoch. + optional uint64 not_after = 7; + // Include app info in the certificate. + bool with_app_info = 8; +} + +message IssueCertResponse { + // The private key the agent generated for this certificate, PEM-encoded. + // Fresh per call; see `IssueCert`. + string key = 1; + // The certificate chain, leaf first, each entry PEM-encoded, exactly as the + // signer returned it. + repeated string certificate_chain = 2; +} + +// --------------------------------------------------------------------------- +// Application keys +// --------------------------------------------------------------------------- + +message GetKeyRequest { + // Caller-chosen domain-separation string identifying the key. Not a DNS + // name -- the certificate fields on `IssueCertRequest` are the ones that + // take those. + // + // Any byte string a proto3 `string` can carry, including one with `:`, `/` + // or NUL in it: the KDF length-prefixes it, so no domain can be confused + // with another. + // + // Derivation is flat. Two domains yield unrelated keys, and `a/b` is not a + // child of `a` in any sense -- there is no BIP-32-style hierarchy and no + // parent key from which a sub-domain's key can be computed. + // + // This replaces v0's `path` plus `purpose`. In v0 only `path` reached the + // KDF and `purpose` was merely echoed into the chain claim; here the one + // field feeds both, alongside `algorithm`. + string domain = 1; + // Key type. Exactly `secp256k1` or `ed25519`. + // + // There is no default and no alias: an empty or unrecognised value is an + // error. v0 defaulted an empty string to secp256k1 and accepted `k256`, + // which meant a typo silently produced a key of the wrong type under a name + // the caller thought meant something else. + // + // Names the key type only. v1 has no signing modes, because it has no + // signing method: v0's `secp256k1_prehashed` was a mode wearing an + // algorithm's name, and it does not appear here. + string algorithm = 2; +} + +message GetKeyResponse { + // The derived private key: 32 raw bytes for both supported algorithms. + bytes key = 1; + // The corresponding public key. SEC1 compressed, 33 bytes, for secp256k1; + // 32 raw bytes for ed25519. This is the exact byte string the signature + // chain's first link commits to, so a relying party never has to re-derive + // it from `key` to check the chain. + bytes public_key = 2; + // Two links, in order: + // [0] the app root key's secp256k1 signature over the v1 key claim, which + // binds algorithm, domain and `public_key` -- see `docs/guest-api-v1.md` + // for the encoding and the verification steps; + // [1] the KMS root key's secp256k1 signature over the app root public key. + // + // Link [1] is produced by the KMS and is byte-identical to the one the + // unversioned surface returns. Link [0] is not: its claim encoding is new, + // and it is deliberately not one the v0 claim format can produce. + repeated bytes signature_chain = 3; +} + +// --------------------------------------------------------------------------- +// Attestation +// --------------------------------------------------------------------------- + +message AttestRequest { + // Up to 64 bytes of report data, zero-padded on the right to 64. + bytes report_data = 1; + // Also return the boot-time GPU attestation evidence in + // `AttestResponse.boottime_gpu_evidence`. This does not sample the GPU + // now and does not answer `report_data`; see that field. + bool include_boottime_gpu_evidence = 2; +} + +message AttestResponse { + // The versioned dstack attestation. + bytes attestation = 1; + // The complete output nvattest produced at boot: a JSON document owned by + // NVIDIA's nvattest, passed through unparsed. Populated only when the + // request asked for it and boot-time GPU attestation output exists. + // + // Not bound to `report_data`: nvattest ran at boot against its own nonce, so + // a fresh `report_data` says nothing about it. Bind it by replaying the + // runtime event log and comparing sha256 of these exact UTF-8 bytes against + // the `evidence_sha256` field of the measured `gpu-attestation` event. + // + // This is a historical statement about the boot, not a live one: it does not + // prove the GPU is still attached. Sampling the GPU at attestation time + // would not fix that -- an NVIDIA report binds the device and a nonce but + // not the TD the device is attached to, so a fresh report can be relayed + // from a genuine remote GPU. Only TDISP/TEE-IO device binding closes that. + string boottime_gpu_evidence = 2; +} + +message AttestGpuRequest { + // Exactly 32 bytes of caller-chosen challenge, passed to the GPU verbatim. + // + // SPDM fixes the evidence nonce at 32 bytes, and dstack applies no transform + // so a caller can compare these bytes directly against the `eat_nonce` claim + // rather than reversing a hash. To bind a longer challenge, hash it + // yourself. + bytes nonce = 1; +} + +message AttestGpuResponse { + // Vendor-native evidence bundles. The caller must select a verifier using + // `vendor` and `format`, then verify the signature, certificate chain, + // measurements, and the nonce embedded in the evidence. + repeated GpuEvidenceBundle bundles = 1; +} + +message GpuEvidenceBundle { + // Stable GPU vendor identifier, for example `nvidia`, `amd`, or `intel`. + string vendor = 1; + // Vendor-specific evidence format and version. + string format = 2; + // Opaque vendor-native evidence bytes. Do not assume UTF-8 or JSON. + bytes evidence = 3; +} + +// --------------------------------------------------------------------------- +// Identity +// --------------------------------------------------------------------------- + +message InfoRequest {} + +// Identity and configuration. Not attestation. +// +// Everything here is what this application *is*: who it is, what it was +// configured with, and where it runs. Nothing here is evidence, and nothing +// here should be trusted on its own -- it arrives over a local socket with no +// quote behind it. +// +// That is why the measurement registers and the event log are absent. v0's +// `AppInfo.tcb_info` carried MRTD, RTMR0-3 and the event log as a JSON blob, +// which invited relying parties to read measurements out of an unattested +// response. Those values belong to `Attest`, whose `VersionedAttestation` +// carries them quote-backed. Ask `Attest` and verify, or do not use them. +// +// `mr_aggregated`, `os_image_hash` and `compose_hash` are the exception, and a +// deliberate one: they identify *which* application and image this is, which is +// the question `Info` answers. They are typed bytes here rather than hex +// strings inside a JSON blob, and each appears exactly once -- v0 returned all +// three both as top-level `AppInfo` fields and again, hex-encoded, inside +// `tcb_info`, two encodings of one fact and free to disagree. They are still +// unattested, and a relying party still confirms them against an attestation. +// +// `app_cert` is gone as well. It was a self-issued demo certificate the agent +// minted for a dashboard, proved nothing, and had no business on a key API. +message InfoResponse { + // Application -- what is deployed. + + // App ID. + bytes app_id = 1; + // App name, from app-compose. + string app_name = 2; + // Compose hash. Adjacent to the document it commits to, so a caller can see + // at a glance which field the hash covers. + bytes compose_hash = 3; + // The app-compose document, as a JSON document owned by the app-compose + // schema (`docs/normalized-app-compose.md`). + // + // These are the verbatim bytes that were deployed, and `compose_hash` is + // sha256 over exactly these bytes. Do not parse and re-serialize before + // hashing: key order, whitespace and unknown fields all change the digest, + // and that digest is what gets whitelisted on chain. + // + // Served directly rather than nested inside another JSON string, which is + // what v0 did via `tcb_info`. + string app_compose = 4; + + // Instance -- which running copy. + + // App instance ID. + bytes instance_id = 5; + + // Platform -- what it runs on. + + // Device ID. Identifies the host machine, not this instance. + bytes device_id = 6; + // OS image hash. + bytes os_image_hash = 7; + // Aggregated measurement register value. + bytes mr_aggregated = 8; + // The VM's hardware configuration, as a JSON document produced and owned by + // the VMM. The guest agent passes it through unparsed. + string vm_config = 9; + // The key provider that supplied this app's keys, as a JSON document owned + // by dstack-util (`{"name": ..., "id": ...}`), passed through unparsed. + string key_provider_info = 10; + // Cloud provider sys_vendor, for example "Google". + string cloud_vendor = 11; + // Cloud provider product_name, for example "Google Compute Engine". + string cloud_product = 12; +} + +message VersionRequest {} + +message VersionResponse { + // dstack version. + string version = 1; + // Git revision. + string rev = 2; +} + +// --------------------------------------------------------------------------- +// The external surface +// --------------------------------------------------------------------------- + +// The versioned external guest agent service. +// +// Served on the external listener at `/prpc/v1`, alongside the closed +// unversioned `Worker` at `/prpc`. +// +// Anyone who can route to the CVM can call this, so nothing here returns key +// material or lets a caller choose what gets signed. `AttestAppKey` names a key +// by algorithm alone and attests the public key the agent derives. +service WorkerV1 { + // Return this application's identity and configuration. + // + // The same `InfoResponse` the internal surface returns, minus what the app + // asked to keep private: unless the app-compose sets `public_tcbinfo`, the + // three document fields (`app_compose`, `vm_config`, `key_provider_info`) + // come back empty. Identity and the measurement hashes are always present, + // which is what the unversioned `Worker.Info` also did. + rpc Info(InfoRequest) returns (InfoResponse) {} + + // Return the guest agent version. + rpc Version(VersionRequest) returns (VersionResponse) {} + + // Attest a key the application derived. + // + // Replaces `Worker.GetAttestationForAppKey`, which returned a + // `GetQuoteResponse` and so could not answer on a platform without Intel TDX. + // Returns what `DstackGuestV1.Attest` returns; that method cannot serve this + // case, because it takes report data from the caller and an external verifier + // does not know the app key's public key until the agent derives it. + rpc AttestAppKey(AttestAppKeyRequest) returns (AttestResponse) {} + + // Report whether the app is serving. Polled by the gateway to decide whether + // this instance should be in its app's load-balancing rotation. + // + // Answers from a cache the agent refreshes on its own timer, so the cost of + // one call is a lock and a clone however many gateway nodes are polling. + // + // Only instances that asked for gating (`RegisterCvmRequest.health_check`) + // are ever polled. That flag, not this method's absence, is what protects an + // older image: the gateway counts *any* failed poll as unhealthy, so probing + // to discover support would blackhole every guest agent that predates this. + rpc Health(HealthRequest) returns (HealthResponse) {} +} + +message AttestAppKeyRequest { + // Key type. Exactly `secp256k1` or `ed25519`, as in `GetKeyRequest`. + string algorithm = 1; +} + +message HealthRequest {} + +// One container that is not reporting healthy. +message ContainerHealth { + // Container name, without the leading slash Docker prepends. + string name = 1; + // Docker health state: "starting" or "unhealthy". A healthy container is + // never listed. A string rather than an enum so it stays readable as JSON. + string status = 2; +} + +// Aggregate application health, as the agent last determined it. +message HealthResponse { + // False when the app's own health file says so or has gone stale, or -- when + // no health file was declared -- when any container that declares a Compose + // `healthcheck` is not running and healthy. + bool healthy = 1; + // The containers that made `healthy` false, so an operator reading gateway + // logs can tell which one is holding the instance out of rotation. Purely + // diagnostic -- the gateway routes on `healthy` alone. + repeated ContainerHealth unhealthy = 2; + // Set when the agent could not see the app at all for several refreshes in + // a row (container runtime unreachable, permission denied). `healthy` is + // false in that case: an agent that cannot see the app is in no position to + // vouch for it. + // + // Reported in the response rather than as an RPC error on purpose. prpc + // answers both "no such method" and "the handler failed" with HTTP 400 and + // drops the message, so an error here would be indistinguishable from an + // agent too old to know this method at all. + string error = 3; +} diff --git a/dstack/guest-agent/rpc/src/generated.rs b/dstack/guest-agent/rpc/src/generated.rs index 28748082f..c968a6411 100644 --- a/dstack/guest-agent/rpc/src/generated.rs +++ b/dstack/guest-agent/rpc/src/generated.rs @@ -4,3 +4,14 @@ pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/file_descriptor_set.bin")); include!(concat!(env!("OUT_DIR"), "/dstack_guest.rs")); + +/// The `dstack.guest.v1` package. +/// +/// Kept in its own module because both packages define types with the same +/// names on purpose -- `SignRequest`, `GpuEvidenceBundle` -- and the two +/// surfaces must stay independently evolvable. A caller names the surface it +/// wants: `dstack_guest_agent_rpc::SignRequest` is the frozen unversioned one, +/// `dstack_guest_agent_rpc::v1::SignRequest` is the v1 one. +pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/dstack.guest.v1.rs")); +} From 6d88699c90ee2539a6f70574cbea5d8fd42e4852 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 02:50:32 -0700 Subject: [PATCH 2/7] feat(guest-agent): serve dstack.guest.v1 and close the unversioned surfaces --- .../src/bin/dstack-kms-sign-cert-fixture.rs | 5 +- dstack/gateway/src/distributed_certbot.rs | 18 +- dstack/gateway/src/proxy/health_check.rs | 69 +- dstack/guest-agent/rpc/proto/agent_rpc.proto | 176 +---- dstack/guest-agent/src/container_health.rs | 2 +- dstack/guest-agent/src/health.rs | 2 +- dstack/guest-agent/src/lib.rs | 1 + dstack/guest-agent/src/rpc_service.rs | 427 +++++----- dstack/guest-agent/src/rpc_service_v1.rs | 733 ++++++++++++++++++ dstack/guest-agent/src/rpc_service_v1/keys.rs | 434 +++++++++++ dstack/guest-agent/src/server.rs | 12 + .../kms/src/main_service/upgrade_authority.rs | 11 +- 12 files changed, 1464 insertions(+), 426 deletions(-) create mode 100644 dstack/guest-agent/src/rpc_service_v1.rs create mode 100644 dstack/guest-agent/src/rpc_service_v1/keys.rs diff --git a/dstack/cert-client/src/bin/dstack-kms-sign-cert-fixture.rs b/dstack/cert-client/src/bin/dstack-kms-sign-cert-fixture.rs index e2a09767c..3451c5535 100644 --- a/dstack/cert-client/src/bin/dstack-kms-sign-cert-fixture.rs +++ b/dstack/cert-client/src/bin/dstack-kms-sign-cert-fixture.rs @@ -3,7 +3,7 @@ //! Generate a v2 KMS CSR whose key is bound to fresh guest attestation. use anyhow::{Context, Result}; -use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, AttestArgs}; +use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, RawQuoteArgs}; use http_client::prpc::PrpcClient; use ra_tls::{ attestation::{PlatformEvidence, QuoteContentType, VersionedAttestation}, @@ -31,9 +31,8 @@ async fn main() -> Result<()> { let address = dstack_types::dstack_agent_address(); let client = DstackGuestClient::new(PrpcClient::new(address)); let response = client - .attest(AttestArgs { + .attest(RawQuoteArgs { report_data: report_data.to_vec(), - include_boottime_gpu_evidence: false, }) .await .context("failed to obtain key-bound guest attestation")?; diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs index 734ab954c..b9193691d 100644 --- a/dstack/gateway/src/distributed_certbot.rs +++ b/dstack/gateway/src/distributed_certbot.rs @@ -12,7 +12,7 @@ use std::time::Duration; use anyhow::{bail, Context, Result}; use certbot::{AcmeClient, Dns01Client}; -use dstack_guest_agent_rpc::{AttestArgs, RawQuoteArgs}; +use dstack_guest_agent_rpc::RawQuoteArgs; use ra_tls::attestation::QuoteContentType; use ra_tls::rcgen::KeyPair; use tokio::sync::Mutex; @@ -658,13 +658,7 @@ impl DistributedCertBot { }; // Get attestation - let attestation_str = match agent - .attest(AttestArgs { - report_data, - include_boottime_gpu_evidence: false, - }) - .await - { + let attestation_str = match agent.attest(RawQuoteArgs { report_data }).await { Ok(resp) => serde_json::to_string(&resp).unwrap_or_default(), Err(err) => { warn!("failed to get attestation for ACME account: {err:?}"); @@ -740,13 +734,7 @@ impl DistributedCertBot { }; // Get attestation - let attestation = match agent - .attest(AttestArgs { - report_data, - include_boottime_gpu_evidence: false, - }) - .await - { + let attestation = match agent.attest(RawQuoteArgs { report_data }).await { Ok(resp) => serde_json::to_string(&resp).unwrap_or_default(), Err(err) => { warn!(domain, "failed to get attestation: {err:?}"); diff --git a/dstack/gateway/src/proxy/health_check.rs b/dstack/gateway/src/proxy/health_check.rs index c364fd3d1..5f6a913bc 100644 --- a/dstack/gateway/src/proxy/health_check.rs +++ b/dstack/gateway/src/proxy/health_check.rs @@ -31,7 +31,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::net::Ipv4Addr; use std::time::Duration; -use dstack_guest_agent_rpc::worker_client::WorkerClient; +use dstack_guest_agent_rpc::v1::worker_v1_client::WorkerV1Client; use futures::StreamExt; use http_client::ConnectionReuse; use tokio::time::MissedTickBehavior; @@ -317,8 +317,8 @@ fn apply_hysteresis( /// than as a verdict; [`apply_hysteresis`] decides when enough of them in a row /// amount to one. async fn poll_instance(ip: Ipv4Addr, agent_port: u16, timeout: Duration) -> PollResult { - let client = WorkerClient::new(prober_transport(ip, agent_port)); - let response = match tokio::time::timeout(timeout, client.health()).await { + let client = WorkerV1Client::new(prober_transport(ip, agent_port)); + let response = match tokio::time::timeout(timeout, client.health(Default::default())).await { Err(_) => { return PollResult::unreachable(format!("health poll timed out after {timeout:?}")) } @@ -348,7 +348,11 @@ async fn poll_instance(ip: Ipv4Addr, agent_port: u16, timeout: Duration) -> Poll /// no real request can reach. Only the connection is unshared; the client /// itself is process-wide. See `http_client::ConnectionReuse`. fn prober_transport(ip: Ipv4Addr, agent_port: u16) -> http_client::prpc::PrpcClient { - let url = format!("http://{ip}:{agent_port}/prpc"); + // `/prpc/v1`, not `/prpc`: `Health` is a `WorkerV1` method. Version skew is + // safe in both directions. A pre-0.6 gateway never polls at all, and only a + // 0.6+ agent can opt in via `RegisterCvmRequest.health_check`, so no agent + // that lacks this path is ever asked for it. + let url = format!("http://{ip}:{agent_port}/prpc/v1"); super::guest_agent_client(url, ConnectionReuse::Fresh) } @@ -362,7 +366,7 @@ const MAX_REASON_BYTES: usize = 512; const MAX_NAMED_CONTAINERS: usize = 8; /// Summarize an agent's "no" for the log line. The caller sanitizes. -fn describe_unhealthy(response: &dstack_guest_agent_rpc::HealthResponse) -> String { +fn describe_unhealthy(response: &dstack_guest_agent_rpc::v1::HealthResponse) -> String { if !response.error.is_empty() { return format!("agent could not determine app health: {}", response.error); } @@ -489,6 +493,37 @@ mod tests { port } + /// The poller must ask the versioned surface. `Health` lives on `WorkerV1` + /// at `/prpc/v1`; the unversioned `Worker` at `/prpc` is closed at v0.5.11 + /// and has no such method, so a poller pointed there gets a 400 that + /// `apply_hysteresis` eventually turns into "unhealthy" for every instance + /// in the fleet at once. + #[tokio::test] + async fn the_poller_asks_the_v1_surface() { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("bind"); + let port = listener.local_addr().expect("addr").port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut buf = [0u8; 1024]; + let n = socket.read(&mut buf).await.unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..n]).into_owned(); + let _ = tx.send(request.lines().next().unwrap_or_default().to_string()); + }); + + let _ = poll_instance(Ipv4Addr::LOCALHOST, port, Duration::from_secs(5)).await; + + let request_line = rx.await.expect("the poller sent no request"); + assert!( + request_line.contains("/prpc/v1/Health"), + "expected the v1 Health path, got: {request_line}" + ); + } + async fn poll_fake(answer: Option>) -> PollResult { let port = fake_agent(answer).await; poll_instance(Ipv4Addr::LOCALHOST, port, Duration::from_secs(5)).await @@ -636,7 +671,7 @@ mod tests { /// `describe_unhealthy` composes it and the constructor bounds it, and only /// the pair is ever what an operator sees -- asserting on `describe_unhealthy` /// alone would pass with the sanitizing dropped. - fn unhealthy_reason(response: &dstack_guest_agent_rpc::HealthResponse) -> String { + fn unhealthy_reason(response: &dstack_guest_agent_rpc::v1::HealthResponse) -> String { match PollResult::unhealthy(describe_unhealthy(response)) { PollResult::Unhealthy(reason) => reason, other => panic!("expected an unhealthy verdict, got {other:?}"), @@ -726,7 +761,7 @@ mod tests { #[test] fn an_agent_error_is_described_rather_than_dropped() { - let response = dstack_guest_agent_rpc::HealthResponse { + let response = dstack_guest_agent_rpc::v1::HealthResponse { healthy: false, unhealthy: vec![], error: "failed to connect to docker".to_string(), @@ -736,14 +771,14 @@ mod tests { #[test] fn unhealthy_containers_are_named() { - let response = dstack_guest_agent_rpc::HealthResponse { + let response = dstack_guest_agent_rpc::v1::HealthResponse { healthy: false, unhealthy: vec![ - dstack_guest_agent_rpc::ContainerHealth { + dstack_guest_agent_rpc::v1::ContainerHealth { name: "web".to_string(), status: "starting".to_string(), }, - dstack_guest_agent_rpc::ContainerHealth { + dstack_guest_agent_rpc::v1::ContainerHealth { name: "db".to_string(), status: "unhealthy".to_string(), }, @@ -761,9 +796,9 @@ mod tests { /// forge a log line or repaint a terminal. #[test] fn control_characters_from_an_agent_never_reach_a_log_line() { - let response = dstack_guest_agent_rpc::HealthResponse { + let response = dstack_guest_agent_rpc::v1::HealthResponse { healthy: false, - unhealthy: vec![dstack_guest_agent_rpc::ContainerHealth { + unhealthy: vec![dstack_guest_agent_rpc::v1::ContainerHealth { name: "web\n2026-01-01 INFO forged".to_string(), status: "\x1b[31mstarting\r".to_string(), }], @@ -783,7 +818,7 @@ mod tests { /// the rendering of everything after it. #[test] fn line_separators_and_bidi_overrides_are_stripped_too() { - let response = dstack_guest_agent_rpc::HealthResponse { + let response = dstack_guest_agent_rpc::v1::HealthResponse { healthy: false, unhealthy: vec![], error: "web\u{2028}Jan 01 INFO ok\u{202E}desrever".to_string(), @@ -797,7 +832,7 @@ mod tests { /// emitted every time the verdict flips. #[test] fn an_oversized_reason_is_truncated() { - let response = dstack_guest_agent_rpc::HealthResponse { + let response = dstack_guest_agent_rpc::v1::HealthResponse { healthy: false, unhealthy: vec![], error: "x".repeat(64 * 1024), @@ -848,10 +883,10 @@ mod tests { /// A thousand containers must not become a thousand-entry log line. #[test] fn only_the_first_few_containers_are_named() { - let response = dstack_guest_agent_rpc::HealthResponse { + let response = dstack_guest_agent_rpc::v1::HealthResponse { healthy: false, unhealthy: (0..50) - .map(|index| dstack_guest_agent_rpc::ContainerHealth { + .map(|index| dstack_guest_agent_rpc::v1::ContainerHealth { name: format!("svc-{index}"), status: "starting".to_string(), }) @@ -868,7 +903,7 @@ mod tests { /// reason an operator can act on. #[test] fn an_unhealthy_response_naming_nothing_still_has_a_reason() { - let response = dstack_guest_agent_rpc::HealthResponse { + let response = dstack_guest_agent_rpc::v1::HealthResponse { healthy: false, unhealthy: vec![], error: String::new(), diff --git a/dstack/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto index 5d043a129..eb1a62bc9 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc.proto @@ -36,10 +36,22 @@ service Tappd { // The service for the dstack guest agent. // -// This unversioned surface is frozen for wire compatibility with v0.5.x -// clients: no field renumbering, no removals, and no semantic changes to -// existing methods. New functionality goes to the upcoming `dstack.guest.v1` -// package instead. +// This unversioned surface is CLOSED. It is exactly the v0.5.11 surface and +// stays that way: no additions, no renumbering, no removals, and no semantic +// changes to existing methods. It exists so a v0.5.x client keeps working +// against a 0.6 agent unchanged. +// +// Every new capability goes to `dstack.guest.v1` (agent_rpc_v1.proto), served +// at `/v1` on the same socket. Do not add a method here, and do not add a +// field to a message here, even a wire-compatible one -- "frozen except for +// additions" is how this surface acquired `GpuInfo`, `AttestGpu` and +// `AttestArgs` between v0.5.11 and 0.6.0, none of which ever shipped and all +// of which have been moved to v1. +// +// Two behaviour-only changes are sanctioned and do not alter the wire shape: +// `GetQuote` now fails on a platform without Intel TDX instead of returning an +// empty quote, and `GetTlsKey` rejects a not_before that is not earlier than +// not_after. `EmitEvent` always fails; see its doc comment. service DstackGuest { // Derives a cryptographic key from the specified key path. // Returns the derived key along with its TLS certificate chain. @@ -58,7 +70,7 @@ service DstackGuest { // Generates a versioned attestation with the given report data. // Returns a dstack-defined attestation format that supports different attestation modes across platforms. - rpc Attest(AttestArgs) returns (AttestResponse) {} + rpc Attest(RawQuoteArgs) returns (AttestResponse) {} // Removed in v0.6.0: always fails. Runtime RTMR3 events are system-owned now, // so an app can no longer extend them. @@ -71,22 +83,6 @@ service DstackGuest { // Get app info rpc Info(google.protobuf.Empty) returns (AppInfo) {} - // Get GPU information collected during boot. - rpc GpuInfo(google.protobuf.Empty) returns (GpuInfoResponse) {} - - // Collect GPU attestation evidence now, against a nonce the caller chooses. - // - // This answers "is the device I can talk to right now a genuine, CC-enabled - // NVIDIA GPU that signs my challenge", which `GpuInfo` cannot: that returns a - // record written at boot. Use it after anything that may have reinitialised - // the GPU -- a driver reload leaves a device that responds to NVML but can no - // longer attest -- and before submitting work you care about. - // - // Returns vendor-native evidence, not a local verdict, so a relying party can - // appraise it with its own verifier. Evidence still does not bind the GPU to - // this TD; see `AttestGpuResponse.bundles`. - rpc AttestGpu(AttestGpuArgs) returns (AttestGpuResponse) {} - // Sign a payload rpc Sign(SignRequest) returns (SignResponse) {} @@ -209,22 +205,15 @@ message TdxQuoteArgs { message RawQuoteArgs { // 64 bytes of report data bytes report_data = 1; -} -// The request to get a versioned attestation -message AttestArgs { - // 64 bytes of report data - bytes report_data = 1; - // Field 2 and 3 carried `include_ccel` and `include_preimages` while this RPC - // took a `RawQuoteArgs`. Both were bools, so reusing either number here would - // make a pre-0.6.0 client's `include_ccel = true` arrive as a request for - // something else entirely. - reserved 2, 3; - reserved "include_ccel", "include_preimages"; - // Also return the boot-time GPU attestation evidence in - // `boottime_gpu_evidence`. This does not sample the GPU now and does not - // answer `report_data`; see that field. - bool include_boottime_gpu_evidence = 4; + // Between v0.5.11 and 0.6.0, `Attest` briefly took its own `AttestArgs` + // carrying `include_ccel` (2), `include_preimages` (3) and + // `include_boottime_gpu_evidence` (4). None of that shipped in a release, + // and `Attest` takes this message again, but an interim `next` build in + // somebody's dev environment may still send those numbers. Reserved so a + // future field cannot silently absorb one. + reserved 2, 3, 4; + reserved "include_ccel", "include_preimages", "include_boottime_gpu_evidence"; } message TdxQuoteResponse { @@ -243,54 +232,12 @@ message TdxQuoteResponse { message AttestResponse { // The attestation bytes attestation = 1; - // Complete JSON output produced by nvattest at boot, the same bytes `GpuInfo` - // serves. Only `DstackGuest.Attest` populates it, and only when the request set - // `include_boottime_gpu_evidence` and boot-time GPU attestation output exists. - // - // Not bound to `report_data`: nvattest ran at boot against its own nonce, so a - // fresh `report_data` says nothing about it. Bind it by replaying the runtime - // event log and comparing sha256 of these exact UTF-8 bytes against the - // `evidence_sha256` field of the measured `gpu-attestation` event. - // - // This is a historical statement about the boot, not a live one: it does not - // prove the GPU is still attached. Sampling the GPU at attestation time would - // not fix that -- an NVIDIA report binds the device and a nonce but not the TD - // the device is attached to, so a fresh report can be relayed from a genuine - // remote GPU. Only TDISP/TEE-IO device binding closes that. - string boottime_gpu_evidence = 2; -} - -message AttestGpuArgs { - // Exactly 32 bytes of caller-chosen challenge, passed to the GPU verbatim. - // - // SPDM fixes the evidence nonce at 32 bytes, and dstack applies no transform - // so a caller can compare these bytes directly against the `eat_nonce` claim - // rather than reversing a hash. To bind a longer challenge, hash it yourself. - bytes nonce = 1; -} -message AttestGpuResponse { - // Vendor-native evidence bundles. The caller must select a verifier using - // `vendor` and `format`, then verify the signature, certificate chain, - // measurements, and the nonce embedded in the evidence. - repeated GpuEvidenceBundle bundles = 1; -} - -message GpuEvidenceBundle { - // Stable GPU vendor identifier, for example `nvidia`, `amd`, or `intel`. - string vendor = 1; - - // Vendor-specific evidence format and version. - string format = 2; - - // Opaque vendor-native evidence bytes. Do not assume UTF-8 or JSON. - bytes evidence = 3; -} - -message GpuInfoResponse { - // Complete JSON output produced by nvattest. Empty when no boot-time GPU - // attestation output is available. - string attestation = 1; + // Field 2 briefly carried `boottime_gpu_evidence` on `next` and never + // shipped. `dstack.guest.v1`'s AttestResponse serves those bytes as + // `boottime_gpu_evidence_json`. + reserved 2; + reserved "boottime_gpu_evidence"; } message GetQuoteResponse { @@ -355,61 +302,24 @@ message WorkerVersion { string rev = 2; } -// One container that is not reporting healthy. -message ContainerHealth { - // Container name, without the leading slash Docker prepends. - string name = 1; - // Docker health state: "starting" or "unhealthy". A healthy container is - // never listed. A string rather than an enum so it stays readable as JSON. - string status = 2; -} - -// Aggregate application health, as the agent last determined it. -message HealthResponse { - // False when the app's own health file says so or has gone stale, or -- when - // no health file was declared -- when any container that declares a Compose - // `healthcheck` is not running and healthy. - bool healthy = 1; - // The containers that made `healthy` false, so an operator reading gateway - // logs can tell which one is holding the instance out of rotation. Purely - // diagnostic -- the gateway routes on `healthy` alone. - repeated ContainerHealth unhealthy = 2; - // Set when the agent could not see the app at all for several refreshes in - // a row (container runtime unreachable, permission denied). `healthy` is - // false in that case: an agent that cannot see the app is in no position to - // vouch for it. - // - // Reported in the response rather than as an RPC error on purpose. prpc - // answers both "no such method" and "the handler failed" with HTTP 400 and - // drops the message, so an error here would be indistinguishable from an - // agent too old to know this method at all. - string error = 3; -} - +// The external guest agent service. +// +// CLOSED, like `DstackGuest`: exactly the v0.5.11 surface, served at `/prpc` +// on the external listener. New capability goes to `dstack.guest.v1`'s +// `WorkerV1`, served at `/prpc/v1` on the same listener. service Worker { // Get app info rpc Info(google.protobuf.Empty) returns (AppInfo) {} // Get the guest agent version rpc Version(google.protobuf.Empty) returns (WorkerVersion) {} - // Attest a key the app derived. - // - // Replaces GetAttestationForAppKey, which returned a GetQuoteResponse and so - // could not answer on a platform without Intel TDX. Returns what - // DstackGuest.Attest returns; that method cannot serve this case, because it - // takes the report data from the caller and an external verifier does not - // know the app key's public key until the agent derives it. - rpc AttestAppKey(AttestAppKeyRequest) returns (AttestResponse) {} - // Report whether the app is serving. Polled by the gateway to decide whether - // this instance should be in its app's load-balancing rotation. - // - // Answers from a cache the agent refreshes on its own timer, so the cost of - // one call is a lock and a clone however many gateway nodes are polling. + // Get attestation. // - // Only instances that asked for gating (`RegisterCvmRequest.health_check`) - // are ever polled. That flag, not this method's absence, is what protects an - // older image: the gateway counts *any* failed poll as unhealthy, so probing - // to discover support would blackhole every guest agent that predates this. - rpc Health(google.protobuf.Empty) returns (HealthResponse) {} + // Legacy and frozen. Returns a `GetQuoteResponse`, so it answers on Intel + // TDX and fails everywhere else -- an external caller on any other platform + // has no way to attest an app key through this method. `WorkerV1.AttestAppKey` + // is the replacement: same request, an attestation that works on every + // platform. + rpc GetAttestationForAppKey(GetAttestationForAppKeyRequest) returns (GetQuoteResponse) {} } message SignRequest { @@ -440,6 +350,6 @@ message VerifyResponse { bool valid = 1; } -message AttestAppKeyRequest { +message GetAttestationForAppKeyRequest { string algorithm = 1; } diff --git a/dstack/guest-agent/src/container_health.rs b/dstack/guest-agent/src/container_health.rs index 2aa65caeb..9c4c69ed3 100644 --- a/dstack/guest-agent/src/container_health.rs +++ b/dstack/guest-agent/src/container_health.rs @@ -37,7 +37,7 @@ use serde::Deserialize; use tokio::process::Command; use tracing::debug; -use dstack_guest_agent_rpc::ContainerHealth; +use dstack_guest_agent_rpc::v1::ContainerHealth; use crate::health::Verdict; diff --git a/dstack/guest-agent/src/health.rs b/dstack/guest-agent/src/health.rs index 35f381634..5a2c94b9d 100644 --- a/dstack/guest-agent/src/health.rs +++ b/dstack/guest-agent/src/health.rs @@ -33,7 +33,7 @@ use dstack_types::HEALTH_FILE_MAX_AGE_SECS; use or_panic::ResultOrPanic; use tracing::{debug, info, warn}; -use dstack_guest_agent_rpc::ContainerHealth; +use dstack_guest_agent_rpc::v1::ContainerHealth; use crate::container_health; diff --git a/dstack/guest-agent/src/lib.rs b/dstack/guest-agent/src/lib.rs index 5ecb7359e..0aee897be 100644 --- a/dstack/guest-agent/src/lib.rs +++ b/dstack/guest-agent/src/lib.rs @@ -14,6 +14,7 @@ mod health; mod http_routes; mod models; pub mod rpc_service; +pub mod rpc_service_v1; mod server; mod socket_activation; diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index 8eb9f1b47..d920008ef 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -2,10 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::{ - path::Path, - sync::{Arc, RwLock}, -}; +use std::sync::{Arc, RwLock}; use anyhow::{Context, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; @@ -15,13 +12,12 @@ use dstack_guest_agent_rpc::{ dstack_guest_server::{DstackGuestRpc, DstackGuestServer}, tappd_server::{TappdRpc, TappdServer}, worker_server::{WorkerRpc, WorkerServer}, - AppInfo, AttestAppKeyRequest, AttestArgs, AttestGpuArgs, AttestGpuResponse, AttestResponse, - DeriveK256KeyResponse, DeriveKeyArgs, EmitEventArgs, GetKeyArgs, GetKeyResponse, - GetQuoteResponse, GetTlsKeyArgs, GetTlsKeyResponse, GpuEvidenceBundle, GpuInfoResponse, - HealthResponse, RawQuoteArgs, SignRequest, SignResponse, TdxQuoteArgs, TdxQuoteResponse, + AppInfo, AttestResponse, DeriveK256KeyResponse, DeriveKeyArgs, EmitEventArgs, + GetAttestationForAppKeyRequest, GetKeyArgs, GetKeyResponse, GetQuoteResponse, GetTlsKeyArgs, + GetTlsKeyResponse, RawQuoteArgs, SignRequest, SignResponse, TdxQuoteArgs, TdxQuoteResponse, VerifyRequest, VerifyResponse, WorkerVersion, }; -use dstack_types::{AppKeys, SysConfig, GPU_ATTESTATION_OUTPUT}; +use dstack_types::{AppKeys, SysConfig}; use ed25519_dalek::ed25519::signature::hazmat::{PrehashSigner, PrehashVerifier}; use ed25519_dalek::{Signer as Ed25519Signer, SigningKey as Ed25519SigningKey, Verifier}; use fs_err as fs; @@ -39,43 +35,19 @@ use rcgen::KeyPair; use ring::rand::{SecureRandom, SystemRandom}; use serde_json::json; use sha3::{Digest, Keccak256}; -use tracing::{error, warn}; +use tracing::error; use crate::{ backend::{PlatformBackend, RealPlatform}, config::Config, }; -fn read_dmi_file(name: &str) -> String { +pub(crate) fn read_dmi_file(name: &str) -> String { fs::read_to_string(format!("/sys/class/dmi/id/{name}")) .map(|s| s.trim().to_string()) .unwrap_or_default() } -/// Read the GPU attestation output saved during boot. Returns an empty string -/// when no output is available (e.g. no GPU attached or attestation disabled). -fn read_gpu_attestation(path: &Path) -> String { - match fs::read_to_string(path) { - Ok(attestation) => attestation, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - warn!("failed to read GPU attestation output: {err:?}"); - } - String::new() - } - } -} - -/// GPU evidence to return alongside an attestation. Opt-in, so a caller that -/// does not care about GPUs neither pays the disk read nor carries the payload. -fn boottime_gpu_evidence(include: bool, path: &Path) -> String { - if include { - read_gpu_attestation(path) - } else { - String::new() - } -} - #[derive(Clone)] pub struct AppState { inner: Arc, @@ -221,25 +193,83 @@ impl AppState { &self.inner.config } - fn health(&self) -> Option<&crate::health::HealthMonitor> { + pub(crate) fn health(&self) -> Option<&crate::health::HealthMonitor> { self.inner.health.as_deref() } - fn quote_response(&self, report_data: [u8; 64]) -> Result { + pub(crate) fn quote_response(&self, report_data: [u8; 64]) -> Result { self.inner .platform .quote_response(report_data, &self.inner.vm_config) } - fn attest_cvm(&self, report_data: [u8; 64]) -> Result> { + pub(crate) fn attest_cvm(&self, report_data: [u8; 64]) -> Result> { self.inner.platform.attest_cvm(report_data)?.to_bytes() } + + /// The application's root secp256k1 key, the root of every derived key and + /// the signer of the first link of every signature chain. + pub(crate) fn app_root_k256_key(&self) -> &[u8] { + &self.inner.keys.k256_key + } + + /// The KMS root key's signature over the app root public key: the second + /// link of every signature chain, produced outside this agent and passed + /// through byte-for-byte on both API surfaces. + pub(crate) fn kms_k256_signature(&self) -> Vec { + self.inner.keys.k256_signature.clone() + } + + /// The VM's hardware configuration, as the VMM produced it. + pub(crate) fn vm_config(&self) -> &str { + &self.inner.vm_config + } + + /// The attestation the agent reports identity from. + pub(crate) fn info_attestation(&self) -> Result { + self.inner.info_attestation() + } + + pub(crate) fn gpu_attestor(&self) -> &crate::gpu_attest::GpuAttestor { + &self.inner.gpu_attestor + } + + pub(crate) async fn issue_cert( + &self, + key: &KeyPair, + config: CertConfigV2, + ) -> Result> { + self.inner.issue_cert(key, config).await + } +} + +/// Generate the fresh P-256 key that backs a certificate the agent issues. +/// +/// Random, not derived: the certificate is minted per call, so there is +/// nothing for a stable key to buy, and a key nobody can re-derive is a +/// smaller thing to hold. Shared by the unversioned `GetTlsKey` and v1's +/// `IssueCert` so the two cannot drift. +pub(crate) fn generate_cert_key() -> Result { + let mut seed = [0u8; 32]; + SystemRandom::new() + .fill(&mut seed) + .context("Failed to generate secure seed")?; + derive_p256_key_pair_from_bytes(&seed, &[]).context("Failed to derive key") } pub struct InternalRpcHandler { state: AppState, } +impl InternalRpcHandler { + /// Only the router constructs this in a running agent; the v1 tests build + /// one directly to assert the unversioned surface still answers as it did. + #[cfg(test)] + pub(crate) fn new(state: AppState) -> Self { + Self { state } + } +} + pub async fn get_info(state: &AppState, external: bool) -> Result { let hide_tcb_info = external && !state.config().app_compose.public_tcbinfo; let versioned_attestation = state.inner.info_attestation()?; @@ -305,7 +335,10 @@ pub async fn get_info(state: &AppState, external: bool) -> Result { }) } -fn validate_cert_validity(not_before: Option, not_after: Option) -> Result<()> { +pub(crate) fn validate_cert_validity( + not_before: Option, + not_after: Option, +) -> Result<()> { if let (Some(not_before), Some(not_after)) = (not_before, not_after) { if not_before >= not_after { anyhow::bail!("not_before must be earlier than not_after"); @@ -317,12 +350,7 @@ fn validate_cert_validity(not_before: Option, not_after: Option) -> Re impl DstackGuestRpc for InternalRpcHandler { async fn get_tls_key(self, request: GetTlsKeyArgs) -> anyhow::Result { validate_cert_validity(request.not_before, request.not_after)?; - let mut seed = [0u8; 32]; - SystemRandom::new() - .fill(&mut seed) - .context("Failed to generate secure seed")?; - let derived_key = - derive_p256_key_pair_from_bytes(&seed, &[]).context("Failed to derive key")?; + let derived_key = generate_cert_key()?; let config = CertConfigV2 { org_name: None, subject: request.subject, @@ -403,29 +431,6 @@ impl DstackGuestRpc for InternalRpcHandler { get_info(&self.state, false).await } - async fn attest_gpu(self, request: AttestGpuArgs) -> Result { - let evidence = self - .state - .inner - .gpu_attestor - .attest(&request.nonce) - .await - .context("GPU attestation failed")?; - Ok(AttestGpuResponse { - bundles: vec![GpuEvidenceBundle { - vendor: "nvidia".to_string(), - format: "nvidia-nvattest-collect-evidence-json-v1".to_string(), - evidence, - }], - }) - } - - async fn gpu_info(self) -> Result { - Ok(GpuInfoResponse { - attestation: read_gpu_attestation(Path::new(GPU_ATTESTATION_OUTPUT)), - }) - } - async fn sign(self, request: SignRequest) -> Result { let algorithm = normalize_algorithm(&request.algorithm); // Use the base algorithm for key derivation (e.g. secp256k1_prehashed -> secp256k1) @@ -526,14 +531,10 @@ impl DstackGuestRpc for InternalRpcHandler { Ok(VerifyResponse { valid }) } - async fn attest(self, request: AttestArgs) -> Result { + async fn attest(self, request: RawQuoteArgs) -> Result { let report_data = pad64(&request.report_data).context("Report data is too long")?; Ok(AttestResponse { attestation: self.state.attest_cvm(report_data)?, - boottime_gpu_evidence: boottime_gpu_evidence( - request.include_boottime_gpu_evidence, - Path::new(GPU_ATTESTATION_OUTPUT), - ), }) } @@ -554,7 +555,7 @@ fn normalize_algorithm(algorithm: &str) -> &str { } } -fn pad64(data: &[u8]) -> Option<[u8; 64]> { +pub(crate) fn pad64(data: &[u8]) -> Option<[u8; 64]> { if data.len() > 64 { return None; } @@ -676,14 +677,6 @@ impl RpcCall for InternalRpcHandlerV0 { } } -fn health_response(verdict: crate::health::Verdict) -> HealthResponse { - HealthResponse { - healthy: verdict.healthy, - unhealthy: verdict.unhealthy, - error: verdict.error, - } -} - pub struct ExternalRpcHandler { state: AppState, } @@ -706,36 +699,22 @@ impl WorkerRpc for ExternalRpcHandler { }) } - async fn health(self) -> Result { - // One lock and one clone. Everything that costs anything happens on the - // agent's own timer in `health`, because this method is served on the - // publicly reachable listener and is polled by every gateway node in - // the cluster: any work done here is work an anonymous caller can ask - // for at an arbitrary rate, multiplied by the operator's fleet size. - let Some(monitor) = self.state.health() else { - // The app did not opt in, so it registered as "do not poll me" and - // no gateway asks. Anything that does ask gets the same answer the - // gateway would have assumed. - return Ok(HealthResponse { - healthy: true, - unhealthy: vec![], - error: String::new(), - }); - }; - // Deliberately infallible: see `HealthResponse.error`. A failure to see - // the app has to come back as a verdict, because an RPC error is - // indistinguishable from an agent that predates this method. - Ok(health_response(monitor.report())) - } - - async fn attest_app_key(self, request: AttestAppKeyRequest) -> Result { + /// Legacy and frozen at the v0.5.11 shape. See the RPC's doc comment in + /// agent_rpc.proto. + /// + /// Returns a `GetQuoteResponse`, which only Intel TDX can fill, so this + /// fails on every other platform exactly as `GetQuote` does. That is the + /// limitation `WorkerV1.AttestAppKey` exists to lift, and it is why this + /// method is not the one new callers should use. + /// + /// The report data comes from the same `app_key_report_data` the v1 method + /// uses, so both attest the same public key for a given algorithm. + async fn get_attestation_for_app_key( + self, + request: GetAttestationForAppKeyRequest, + ) -> Result { let report_data = self.app_key_report_data(&request.algorithm).await?; - Ok(AttestResponse { - attestation: self.state.attest_cvm(report_data)?, - // This method attests a key, not the machine. A caller that wants - // the boot-time GPU evidence asks `Attest` or `GpuInfo` for it. - boottime_gpu_evidence: String::new(), - }) + self.state.quote_response(report_data) } } @@ -747,7 +726,7 @@ impl ExternalRpcHandler { /// until the key is derived here -- which is why attesting an app key needs /// its own method instead of the caller-supplied report data `GetQuote` /// and `Attest` take. - async fn app_key_report_data(&self, algorithm: &str) -> Result<[u8; 64]> { + pub(crate) async fn app_key_report_data(&self, algorithm: &str) -> Result<[u8; 64]> { let algorithm = normalize_algorithm(algorithm); // Prehashing is a signing mode, not a key type: the same secp256k1 key // signs both ways, so derive it under the base name. `Sign` does the @@ -812,14 +791,17 @@ impl RpcCall for ExternalRpcHandler { } #[cfg(test)] -mod tests { +// `pub(crate)` so the v1 handler's tests can build a state from the same +// fixture. Two fixtures would let the two surfaces be tested against different +// app root keys, which is exactly what the cross-version key assertions check. +pub(crate) mod tests { use super::*; use crate::{ backend::PlatformBackend, config::{AppComposeWrapper, Config}, }; use dstack_attest::attestation::AttestationVerifier; - use dstack_guest_agent_rpc::{AttestAppKeyRequest, SignRequest}; + use dstack_guest_agent_rpc::{GetAttestationForAppKeyRequest, SignRequest}; use dstack_types::{AppCompose, AppKeys, EventLogVersion, KeyProvider}; use ed25519_dalek::ed25519::signature::hazmat::PrehashVerifier; use ed25519_dalek::{ @@ -832,41 +814,6 @@ mod tests { use std::convert::TryFrom; use std::io::Write; - #[test] - fn reads_gpu_attestation_output_verbatim() { - let mut output = tempfile::NamedTempFile::new().unwrap(); - let attestation = r#"{"result_code":0,"claims":[]}"#; - output.write_all(attestation.as_bytes()).unwrap(); - output.flush().unwrap(); - - assert_eq!(read_gpu_attestation(output.path()), attestation); - } - - #[test] - fn attest_returns_boottime_gpu_evidence_only_when_requested() { - let mut output = tempfile::NamedTempFile::new().unwrap(); - let evidence = r#"{"result_code":0,"claims":[]}"#; - output.write_all(evidence.as_bytes()).unwrap(); - output.flush().unwrap(); - - assert_eq!(boottime_gpu_evidence(true, output.path()), evidence); - assert_eq!(boottime_gpu_evidence(false, output.path()), ""); - } - - #[test] - fn missing_gpu_attestation_output_reads_as_empty() { - let dir = tempfile::tempdir().unwrap(); - assert_eq!(read_gpu_attestation(&dir.path().join("missing")), ""); - } - - fn app_key_report_data(response: &AttestResponse) -> [u8; 64] { - VersionedAttestation::from_bytes(&response.attestation) - .expect("failed to decode attestation") - .into_v1() - .report_data() - .expect("attestation carries no report data") - } - fn extract_pubkey_from_report_data(report_data: &[u8], prefix: &str) -> Result> { let end = report_data .iter() @@ -883,13 +830,31 @@ mod tests { } } - async fn setup_test_state() -> (AppState, tempfile::NamedTempFile) { + pub(crate) async fn setup_test_state() -> (AppState, tempfile::NamedTempFile) { setup_test_state_with_platform(None).await } /// The same state, with the fixture's platform evidence swapped out. /// `None` keeps the fixture's Intel TDX evidence. - async fn setup_test_state_with_platform( + /// The same state with the app having opted into publishing its TCB info, + /// which is what unlocks the document fields on the external surface. + pub(crate) async fn setup_test_state_with_public_tcbinfo() -> (AppState, tempfile::NamedTempFile) + { + let (state, guard) = setup_test_state().await; + let mut config = state.config().clone(); + config.app_compose.app_compose.public_tcbinfo = true; + config.app_compose.raw = r#"{"name":"test"}"#.to_string(); + let mut inner = Arc::try_unwrap(state.inner).ok().expect("sole owner"); + inner.config = config; + ( + AppState { + inner: Arc::new(inner), + }, + guard, + ) + } + + pub(crate) async fn setup_test_state_with_platform( platform: Option, ) -> (AppState, tempfile::NamedTempFile) { let mut temp_attestation_file = tempfile::NamedTempFile::new().unwrap(); @@ -1110,14 +1075,10 @@ pNs85uhOZE8z2jr8Pg== let response = handler.sign(request).await.unwrap(); - let attestation_response = ExternalRpcHandler::new(state) - .attest_app_key(AttestAppKeyRequest { - algorithm: "ed25519".to_string(), - }) + let report_data = ExternalRpcHandler::new(state) + .app_key_report_data("ed25519") .await .unwrap(); - - let report_data = app_key_report_data(&attestation_response); let pk_bytes = extract_pubkey_from_report_data(&report_data, "dip1::ed25519-pk:").unwrap(); let public_key = Ed25519VerifyingKey::try_from(pk_bytes.as_slice()).unwrap(); @@ -1139,14 +1100,10 @@ pNs85uhOZE8z2jr8Pg== let response = handler.sign(request).await.unwrap(); - let attestation_response = ExternalRpcHandler::new(state) - .attest_app_key(AttestAppKeyRequest { - algorithm: "secp256k1".to_string(), - }) + let report_data = ExternalRpcHandler::new(state) + .app_key_report_data("secp256k1") .await .unwrap(); - - let report_data = app_key_report_data(&attestation_response); let pk_bytes = extract_pubkey_from_report_data(&report_data, "dip1::secp256k1c-pk:").unwrap(); @@ -1172,14 +1129,10 @@ pNs85uhOZE8z2jr8Pg== let response = handler.sign(request).await.unwrap(); - let attestation_response = ExternalRpcHandler::new(state) - .attest_app_key(AttestAppKeyRequest { - algorithm: "secp256k1".to_string(), - }) + let report_data = ExternalRpcHandler::new(state) + .app_key_report_data("secp256k1") .await .unwrap(); - - let report_data = app_key_report_data(&attestation_response); let pk_bytes = extract_pubkey_from_report_data(&report_data, "dip1::secp256k1c-pk:").unwrap(); @@ -1227,49 +1180,60 @@ pNs85uhOZE8z2jr8Pg== assert_eq!(result.unwrap_err().to_string(), "Unsupported algorithm"); } + const ED25519_REPORT_DATA: &str = + "dip1::ed25519-pk:5Pbre1Amf1hrp2V2bbfKlIfxpQb2pJAmrgmhxgVoG9s\0\0\0\0"; + const SECP256K1_REPORT_DATA: &str = + "dip1::secp256k1c-pk:A6t_JdVkVdMAocH3f1f20WGT6JzdntxcXimUtEax8zc9"; + + /// The DIP-1 report data both external surfaces commit to. `WorkerV1` + /// wraps these exact bytes in an attestation and the frozen + /// `GetAttestationForAppKey` wraps them in a TDX quote, so pinning them + /// here pins both. #[tokio::test] - async fn test_attest_app_key_ed25519_success() { + async fn app_key_report_data_matches_its_vectors() { let (state, _guard) = setup_test_state().await; - let handler = ExternalRpcHandler::new(state.clone()); - let request = AttestAppKeyRequest { - algorithm: "ed25519".to_string(), - }; - - let response = handler.attest_app_key(request).await.unwrap(); - - const EXPECTED_REPORT_DATA: &str = - "dip1::ed25519-pk:5Pbre1Amf1hrp2V2bbfKlIfxpQb2pJAmrgmhxgVoG9s\0\0\0\0"; - assert_eq!( - EXPECTED_REPORT_DATA.as_bytes(), - app_key_report_data(&response).as_slice() - ); + for (algorithm, expected) in [ + ("ed25519", ED25519_REPORT_DATA), + ("secp256k1", SECP256K1_REPORT_DATA), + ] { + let report_data = ExternalRpcHandler::new(state.clone()) + .app_key_report_data(algorithm) + .await + .unwrap(); + assert_eq!(expected.as_bytes(), report_data.as_slice(), "{algorithm}"); + } } + /// Prehashing changes how a key signs, not which key it is, so it must + /// commit to the same public key as the plain name. #[tokio::test] - async fn test_attest_app_key_secp256k1_success() { + async fn app_key_report_data_accepts_secp256k1_prehashed() { let (state, _guard) = setup_test_state().await; - let handler = ExternalRpcHandler::new(state.clone()); - let request = AttestAppKeyRequest { - algorithm: "secp256k1".to_string(), - }; - - let response = handler.attest_app_key(request).await.unwrap(); + let prehashed = ExternalRpcHandler::new(state.clone()) + .app_key_report_data("secp256k1_prehashed") + .await + .expect("secp256k1_prehashed must be accepted"); + let plain = ExternalRpcHandler::new(state) + .app_key_report_data("secp256k1") + .await + .unwrap(); + assert_eq!(prehashed, plain); + } - const EXPECTED_REPORT_DATA: &str = - "dip1::secp256k1c-pk:A6t_JdVkVdMAocH3f1f20WGT6JzdntxcXimUtEax8zc9"; - assert_eq!( - EXPECTED_REPORT_DATA.as_bytes(), - app_key_report_data(&response).as_slice() - ); + #[tokio::test] + async fn app_key_report_data_rejects_an_unsupported_algorithm() { + let (state, _guard) = setup_test_state().await; + let result = ExternalRpcHandler::new(state) + .app_key_report_data("ecdsa") + .await; + assert_eq!(result.unwrap_err().to_string(), "Unsupported algorithm"); } + /// The frozen method returns a `GetQuoteResponse`, which only Intel TDX can + /// fill. That limitation is the whole reason `WorkerV1.AttestAppKey` + /// exists, so it has to stay observable here. #[tokio::test] - async fn test_attest_app_key_works_on_non_tdx() { - // The reason this method exists in place of the GetQuoteResponse-shaped - // one it replaces: the external listener has no other way to attest an - // app key, and `Attest` is no substitute -- it is on the internal - // socket, and its caller would have to know the app key's public key in - // advance to build the report data. + async fn get_attestation_for_app_key_is_tdx_only() { let (state, _guard) = setup_test_state_with_platform(Some(PlatformEvidence::SevSnp { report: vec![0u8; 1184], cert_chain: Vec::new(), @@ -1277,64 +1241,29 @@ pNs85uhOZE8z2jr8Pg== })) .await; - // GetQuote is closed on this platform... - let err = state - .quote_response([0x5a; 64]) - .expect_err("GetQuote must fail on a non-TDX platform"); - assert!( - err.to_string().contains("Intel TDX only"), - "unexpected error: {err}" - ); - - // ...and attesting an app key still works. - let response = ExternalRpcHandler::new(state) - .attest_app_key(AttestAppKeyRequest { + let err = ExternalRpcHandler::new(state) + .get_attestation_for_app_key(GetAttestationForAppKeyRequest { algorithm: "ed25519".to_string(), }) .await - .unwrap(); - - const EXPECTED_REPORT_DATA: &str = - "dip1::ed25519-pk:5Pbre1Amf1hrp2V2bbfKlIfxpQb2pJAmrgmhxgVoG9s\0\0\0\0"; - assert_eq!( - EXPECTED_REPORT_DATA.as_bytes(), - app_key_report_data(&response).as_slice() + .expect_err("the frozen method cannot answer without a TDX quote"); + assert!( + err.to_string().contains("Intel TDX only"), + "unexpected error: {err}" ); } #[tokio::test] - async fn test_attest_app_key_accepts_secp256k1_prehashed() { - // Prehashing changes how the key signs, not which key it is, so this - // must attest the same public key `Sign` uses under that name. + async fn get_attestation_for_app_key_answers_on_tdx() { let (state, _guard) = setup_test_state().await; - - let prehashed = ExternalRpcHandler::new(state.clone()) - .attest_app_key(AttestAppKeyRequest { - algorithm: "secp256k1_prehashed".to_string(), - }) - .await - .expect("secp256k1_prehashed must be accepted"); - let plain = ExternalRpcHandler::new(state) - .attest_app_key(AttestAppKeyRequest { - algorithm: "secp256k1".to_string(), + let response = ExternalRpcHandler::new(state) + .get_attestation_for_app_key(GetAttestationForAppKeyRequest { + algorithm: "ed25519".to_string(), }) .await .unwrap(); - - assert_eq!(app_key_report_data(&prehashed), app_key_report_data(&plain)); - } - - #[tokio::test] - async fn test_attest_app_key_unsupported_algorithm_fails() { - let (state, _guard) = setup_test_state().await; - let handler = ExternalRpcHandler::new(state); - let request = AttestAppKeyRequest { - algorithm: "ecdsa".to_string(), // Unsupported algorithm - }; - - let result = handler.attest_app_key(request).await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().to_string(), "Unsupported algorithm"); + assert_eq!(ED25519_REPORT_DATA.as_bytes(), response.report_data); + assert!(!response.quote.is_empty()); } #[test] diff --git a/dstack/guest-agent/src/rpc_service_v1.rs b/dstack/guest-agent/src/rpc_service_v1.rs new file mode 100644 index 000000000..2f0867956 --- /dev/null +++ b/dstack/guest-agent/src/rpc_service_v1.rs @@ -0,0 +1,733 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! The `dstack.guest.v1` API surface. +//! +//! Two handlers, one per trust surface, mirroring the two unversioned ones: +//! +//! - [`V1RpcHandler`] on the internal socket at `/v1`, which hands out key +//! material because only the application itself can reach that socket; +//! - [`ExternalV1RpcHandler`] on the external listener at `/prpc/v1`, which +//! never does, because anyone who can route to the CVM can reach it. +//! +//! The unversioned handlers in [`crate::rpc_service`] keep serving `/` and +//! `/prpc` unchanged; a caller picks a surface by URL and nothing else. +//! +//! This handler shares every backend with the unversioned one -- the same +//! `AppState`, the same certificate client, the same platform attestation and +//! GPU attestor. What it does not share is key derivation and the +//! signature-chain claim, which are new in v1 and live in [`keys`]. +//! +//! v1 serves only what needs the TEE: deriving from the app root key, and +//! attesting. It has no `Sign` and no `Verify`, because a caller that can +//! reach this socket can get the private key from `GetKey` and do both +//! itself. + +use std::path::Path; + +use anyhow::{Context, Result}; +use dstack_guest_agent_rpc::v1::{ + dstack_guest_v1_server::{DstackGuestV1Rpc, DstackGuestV1Server}, + worker_v1_server::{WorkerV1Rpc, WorkerV1Server}, + AttestAppKeyRequest, AttestGpuRequest, AttestGpuResponse, AttestRequest, AttestResponse, + GetKeyRequest, GetKeyResponse, GpuEvidenceBundle, HealthRequest, HealthResponse, InfoRequest, + InfoResponse, IssueCertRequest, IssueCertResponse, VersionRequest, VersionResponse, +}; +use dstack_types::GPU_ATTESTATION_OUTPUT; +use fs_err as fs; +use ra_rpc::{CallContext, RpcCall}; +use ra_tls::cert::CertConfigV2; +use tracing::warn; + +use crate::rpc_service::{ + generate_cert_key, pad64, read_dmi_file, validate_cert_validity, AppState, ExternalRpcHandler, +}; + +pub(crate) mod keys; + +use keys::{Algorithm, AppKey}; + +/// Read the GPU attestation output saved during boot. Returns an empty string +/// when no output is available (e.g. no GPU attached or attestation disabled). +fn read_gpu_attestation(path: &Path) -> String { + match fs::read_to_string(path) { + Ok(attestation) => attestation, + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + warn!("failed to read GPU attestation output: {err:?}"); + } + String::new() + } + } +} + +/// GPU evidence to return alongside an attestation. Opt-in, so a caller that +/// does not care about GPUs neither pays the disk read nor carries the payload. +fn boottime_gpu_evidence(include: bool, path: &Path) -> String { + if include { + read_gpu_attestation(path) + } else { + String::new() + } +} + +/// Build the v1 `Info` response. +/// +/// Built straight from the attestation's app info rather than from the +/// unversioned `AppInfo`, because v1 needs the verbatim app-compose document +/// that v0 only ever exposed nested inside the `tcb_info` JSON string. +/// +/// `hide_documents` applies the app's `public_tcbinfo` choice: with it set, the +/// three document fields come back empty and everything identifying the app +/// stays visible. That is the same line the unversioned `Worker.Info` drew, and +/// it is drawn only on the external listener -- an app cannot need protecting +/// from itself. +fn info_response(state: &AppState, hide_documents: bool) -> Result { + let attestation = state.info_attestation()?.into_v1(); + let app_info = attestation + .decode_app_info(false) + .context("Failed to decode app info")?; + let document = |value: String| if hide_documents { String::new() } else { value }; + Ok(InfoResponse { + app_id: app_info.app_id, + app_name: state.config().app_compose.name.clone(), + compose_hash: app_info.compose_hash, + app_compose: document(state.config().app_compose.raw.clone()), + instance_id: app_info.instance_id, + device_id: app_info.device_id, + os_image_hash: app_info.os_image_hash, + mr_aggregated: app_info.mr_aggregated.to_vec(), + vm_config: document(state.vm_config().to_string()), + key_provider_info: document( + String::from_utf8(app_info.key_provider_info).unwrap_or_default(), + ), + cloud_vendor: read_dmi_file("sys_vendor"), + cloud_product: read_dmi_file("product_name"), + }) +} + +pub struct V1RpcHandler { + state: AppState, +} + +impl V1RpcHandler { + #[cfg(test)] + pub(crate) fn new(state: AppState) -> Self { + Self { state } + } + + /// Derive the key named by `(domain, algorithm)` together with its + /// signature chain. + fn derive(&self, domain: &str, algorithm: &str) -> Result<(AppKey, Vec>)> { + let algorithm = Algorithm::parse(algorithm)?; + let app_root_key = self.state.app_root_k256_key(); + let key = AppKey::derive(app_root_key, domain, algorithm)?; + let chain = vec![ + key.claim_signature(app_root_key)?, + self.state.kms_k256_signature(), + ]; + Ok((key, chain)) + } +} + +impl DstackGuestV1Rpc for V1RpcHandler { + async fn issue_cert(self, request: IssueCertRequest) -> Result { + validate_cert_validity(request.not_before, request.not_after)?; + let key = generate_cert_key()?; + let config = CertConfigV2 { + org_name: None, + subject: request.subject, + subject_alt_names: request.alt_names, + usage_server_auth: request.usage_server_auth, + usage_client_auth: request.usage_client_auth, + ext_quote: request.usage_ra_tls, + ext_app_info: request.with_app_info, + not_after: request.not_after, + not_before: request.not_before, + }; + let certificate_chain = self.state.issue_cert(&key, config).await?; + Ok(IssueCertResponse { + key: key.serialize_pem(), + certificate_chain, + }) + } + + async fn get_key(self, request: GetKeyRequest) -> Result { + let (key, signature_chain) = self.derive(&request.domain, &request.algorithm)?; + Ok(GetKeyResponse { + key: key.secret(), + public_key: key.public_key(), + signature_chain, + }) + } + + async fn attest(self, request: AttestRequest) -> Result { + let report_data = pad64(&request.report_data).context("Report data is too long")?; + Ok(AttestResponse { + attestation: self.state.attest_cvm(report_data)?, + boottime_gpu_evidence: boottime_gpu_evidence( + request.include_boottime_gpu_evidence, + Path::new(GPU_ATTESTATION_OUTPUT), + ), + }) + } + + async fn attest_gpu(self, request: AttestGpuRequest) -> Result { + let evidence = self + .state + .gpu_attestor() + .attest(&request.nonce) + .await + .context("GPU attestation failed")?; + Ok(AttestGpuResponse { + bundles: vec![GpuEvidenceBundle { + vendor: "nvidia".to_string(), + format: "nvidia-nvattest-collect-evidence-json-v1".to_string(), + evidence, + }], + }) + } + + /// Identity and configuration, never attestation. + /// + /// Ungated: the internal socket is reachable only by the application + /// itself, so there is nobody to hide from. The external surface applies + /// `public_tcbinfo`; see [`ExternalV1RpcHandler::info`]. + async fn info(self, _request: InfoRequest) -> Result { + info_response(&self.state, false) + } + + async fn version(self, _request: VersionRequest) -> Result { + Ok(VersionResponse { + version: crate::CARGO_PKG_VERSION.to_string(), + rev: crate::GIT_REV.to_string(), + }) + } +} + +impl RpcCall for V1RpcHandler { + type PrpcService = DstackGuestV1Server; + + fn construct(context: CallContext<'_, AppState>) -> Result { + Ok(V1RpcHandler { + state: context.state.clone(), + }) + } +} + +/// The v1 handler on the external listener. +/// +/// Reachable by anyone who can route to the CVM, so it serves no key material +/// and lets no caller choose what gets signed. +pub struct ExternalV1RpcHandler { + state: AppState, +} + +impl ExternalV1RpcHandler { + #[cfg(test)] + pub(crate) fn new(state: AppState) -> Self { + Self { state } + } +} + +impl WorkerV1Rpc for ExternalV1RpcHandler { + async fn info(self, _request: InfoRequest) -> Result { + let hide = !self.state.config().app_compose.public_tcbinfo; + info_response(&self.state, hide) + } + + async fn version(self, _request: VersionRequest) -> Result { + Ok(VersionResponse { + version: crate::CARGO_PKG_VERSION.to_string(), + rev: crate::GIT_REV.to_string(), + }) + } + + async fn attest_app_key(self, request: AttestAppKeyRequest) -> Result { + // Same report data the frozen `Worker.GetAttestationForAppKey` builds, + // so both surfaces attest the same public key for a given algorithm. + // What changes is the envelope: an attestation answers on every + // platform, a `GetQuoteResponse` only on Intel TDX. + let report_data = ExternalRpcHandler::new(self.state.clone()) + .app_key_report_data(&request.algorithm) + .await?; + Ok(AttestResponse { + attestation: self.state.attest_cvm(report_data)?, + // This method attests a key, not the machine. A caller that wants + // the boot-time GPU evidence asks `Attest` for it. + boottime_gpu_evidence: String::new(), + }) + } + + async fn health(self, _request: HealthRequest) -> Result { + // One lock and one clone. Everything that costs anything happens on the + // agent's own timer in `health`, because this method is served on the + // publicly reachable listener and is polled by every gateway node in + // the cluster: any work done here is work an anonymous caller can ask + // for at an arbitrary rate, multiplied by the operator's fleet size. + let Some(monitor) = self.state.health() else { + // The app did not opt in, so it registered as "do not poll me" and + // no gateway asks. Anything that does ask gets the same answer the + // gateway would have assumed. + return Ok(HealthResponse { + healthy: true, + unhealthy: vec![], + error: String::new(), + }); + }; + // Deliberately infallible: see `HealthResponse.error`. A failure to see + // the app has to come back as a verdict, because an RPC error is + // indistinguishable from an agent that predates this method. + let verdict = monitor.report(); + Ok(HealthResponse { + healthy: verdict.healthy, + unhealthy: verdict.unhealthy, + error: verdict.error, + }) + } +} + +impl RpcCall for ExternalV1RpcHandler { + type PrpcService = WorkerV1Server; + + fn construct(context: CallContext<'_, AppState>) -> Result { + Ok(ExternalV1RpcHandler { + state: context.state.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_service::get_info; + use crate::rpc_service::tests::setup_test_state; + use dstack_guest_agent_rpc::dstack_guest_server::DstackGuestRpc; + use dstack_guest_agent_rpc::GetKeyArgs; + use k256::ecdsa::Signature as K256Signature; + use std::io::Write as _; + + async fn handler() -> (V1RpcHandler, AppState, tempfile::NamedTempFile) { + let (state, guard) = setup_test_state().await; + (V1RpcHandler::new(state.clone()), state, guard) + } + + fn get_key_request(domain: &str, algorithm: &str) -> GetKeyRequest { + GetKeyRequest { + domain: domain.to_string(), + algorithm: algorithm.to_string(), + } + } + + /// The fixture's app root key is what `keys::tests` committed its vectors + /// against. If the two drift, the vectors stop describing what this + /// handler serves and every assertion built on them goes quiet. + #[tokio::test] + async fn the_fixture_root_key_matches_the_committed_vectors() { + let (_, state, _guard) = handler().await; + assert_eq!( + hex::encode(state.app_root_k256_key()), + "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b" + ); + } + + #[tokio::test] + async fn get_key_returns_a_two_link_chain_for_both_algorithms() { + for algorithm in ["secp256k1", "ed25519"] { + let (_, state, _guard) = handler().await; + let key = V1RpcHandler::new(state) + .get_key(get_key_request("wallet", algorithm)) + .await + .unwrap(); + + assert_eq!(key.key.len(), 32); + assert_eq!(key.signature_chain.len(), 2); + // r || s || recovery id. + assert_eq!(key.signature_chain[0].len(), 65); + } + } + + /// The chain's first link verifies under the app root key, over the claim + /// a relying party rebuilds from `(domain, algorithm, public_key)`. + #[tokio::test] + async fn the_chain_claim_verifies_under_the_app_root_key() { + use k256::ecdsa::{RecoveryId, SigningKey, VerifyingKey}; + use sha3::{Digest as _, Keccak256}; + + let (_, state, _guard) = handler().await; + let key = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("wallet", "secp256k1")) + .await + .unwrap(); + + let claim = keys::key_claim(keys::Algorithm::Secp256k1, "wallet", &key.public_key).unwrap(); + let link = &key.signature_chain[0]; + let recovered = VerifyingKey::recover_from_digest( + Keccak256::new_with_prefix(&claim), + &K256Signature::from_slice(&link[..64]).unwrap(), + RecoveryId::from_byte(link[64]).unwrap(), + ) + .unwrap(); + + let app_root = *SigningKey::from_slice(state.app_root_k256_key()) + .unwrap() + .verifying_key(); + assert_eq!(recovered, app_root); + } + + /// The public key in the response is the one the returned private key + /// actually has. An app that signs locally with `key` must land on + /// `public_key`, which is what the chain vouches for. + #[tokio::test] + async fn the_public_key_belongs_to_the_returned_private_key() { + let (_, state, _guard) = handler().await; + + let secp = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("wallet", "secp256k1")) + .await + .unwrap(); + let derived = k256::ecdsa::SigningKey::from_slice(&secp.key).unwrap(); + assert_eq!( + derived.verifying_key().to_sec1_bytes().to_vec(), + secp.public_key + ); + + let ed = V1RpcHandler::new(state) + .get_key(get_key_request("wallet", "ed25519")) + .await + .unwrap(); + let seed: [u8; 32] = ed.key.as_slice().try_into().unwrap(); + let derived = ed25519_dalek::SigningKey::from_bytes(&seed); + assert_eq!(derived.verifying_key().to_bytes().to_vec(), ed.public_key); + } + + #[tokio::test] + async fn rejects_an_empty_or_unknown_algorithm() { + let (_, state, _guard) = handler().await; + for algorithm in ["", "k256", "rsa", "secp256k1_prehashed"] { + let result = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("wallet", algorithm)) + .await; + assert!(result.is_err(), "v1 accepted algorithm {algorithm:?}"); + } + } + + /// v1 keys are new key material. An app that reuses its v0 path as a v1 + /// domain gets a different key, and that is the migration contract, not a + /// bug to paper over. + #[tokio::test] + async fn v1_keys_differ_from_the_unversioned_keys_for_the_same_name() { + let (_, state, _guard) = handler().await; + let v1 = V1RpcHandler::new(state.clone()) + .get_key(get_key_request("wallet", "secp256k1")) + .await + .unwrap(); + let v0 = crate::rpc_service::InternalRpcHandler::new(state) + .get_key(GetKeyArgs { + path: "wallet".to_string(), + purpose: "signing".to_string(), + algorithm: "secp256k1".to_string(), + }) + .await + .unwrap(); + + assert_ne!(v1.key, v0.key); + // The second link is the KMS's, produced outside the agent, so both + // surfaces pass through the same bytes. + assert_eq!(v1.signature_chain[1], v0.signature_chain[1]); + assert_ne!(v1.signature_chain[0], v0.signature_chain[0]); + } + + /// The unversioned handler is untouched by any of this: same fixture, same + /// path, the v0.5.x answer. + #[tokio::test] + async fn the_unversioned_surface_still_serves_its_own_key() { + let (_, state, _guard) = handler().await; + // v0 defaults an empty algorithm to secp256k1 and derives from the + // path alone; v1 rejects the same request outright. + let expected = ra_tls::kdf::derive_key(state.app_root_k256_key(), &[b"test"], 32).unwrap(); + let v0 = crate::rpc_service::InternalRpcHandler::new(state) + .get_key(GetKeyArgs { + path: "test".to_string(), + purpose: "signing".to_string(), + algorithm: String::new(), + }) + .await + .unwrap(); + assert_eq!(v0.key, expected); + } + + /// v1 `Info` reports identity and configuration, and nothing that only an + /// attestation should be trusted for. + #[tokio::test] + async fn info_reports_identity_and_configuration() { + let (_, state, _guard) = handler().await; + let v1 = V1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + let v0 = get_info(&state, false).await.unwrap(); + + // The identity fields carry the same values the unversioned surface + // reports; only the shape changed. + assert_eq!(v1.app_id, v0.app_id); + assert_eq!(v1.instance_id, v0.instance_id); + assert_eq!(v1.device_id, v0.device_id); + assert_eq!(v1.mr_aggregated, v0.mr_aggregated); + assert_eq!(v1.os_image_hash, v0.os_image_hash); + assert_eq!(v1.compose_hash, v0.compose_hash); + assert_eq!(v1.app_name, v0.app_name); + assert_eq!(v1.vm_config, v0.vm_config); + assert_eq!(v1.key_provider_info, v0.key_provider_info); + assert_eq!(v1.cloud_vendor, v0.cloud_vendor); + assert_eq!(v1.cloud_product, v0.cloud_product); + + // The app-compose document is served directly rather than nested in a + // JSON string, which is the one thing v0 could not do. + assert_eq!(v1.app_compose, state.config().app_compose.raw); + } + + /// GPU evidence rides on `Attest`, which is the only way to get it now. + #[tokio::test] + async fn attest_returns_boot_time_gpu_evidence_only_when_asked() { + let mut output = tempfile::NamedTempFile::new().unwrap(); + let evidence = r#"{"result_code":0,"claims":[]}"#; + output.write_all(evidence.as_bytes()).unwrap(); + output.flush().unwrap(); + + assert_eq!(boottime_gpu_evidence(true, output.path()), evidence); + assert_eq!(boottime_gpu_evidence(false, output.path()), ""); + } + + #[test] + fn reads_gpu_attestation_output_verbatim() { + let mut output = tempfile::NamedTempFile::new().unwrap(); + let attestation = r#"{"result_code":0,"claims":[]}"#; + output.write_all(attestation.as_bytes()).unwrap(); + output.flush().unwrap(); + + assert_eq!(read_gpu_attestation(output.path()), attestation); + } + + #[test] + fn missing_gpu_attestation_output_reads_as_empty() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!(read_gpu_attestation(&dir.path().join("missing")), ""); + } + + /// `Attest` is v1's only CVM attestation entry point, and the versioned + /// attestation it returns carries the report data the caller asked for. + #[tokio::test] + async fn attest_reports_the_padded_report_data() { + use ra_tls::attestation::VersionedAttestation; + + let (_, state, _guard) = handler().await; + let response = V1RpcHandler::new(state) + .attest(AttestRequest { + report_data: b"hello".to_vec(), + include_boottime_gpu_evidence: false, + }) + .await + .unwrap(); + + let report_data = VersionedAttestation::from_bytes(&response.attestation) + .unwrap() + .into_v1() + .report_data() + .unwrap(); + assert_eq!(&report_data[..5], b"hello"); + assert!(report_data[5..].iter().all(|b| *b == 0)); + } + + #[tokio::test] + async fn rejects_report_data_longer_than_64_bytes() { + let (_, state, _guard) = handler().await; + let err = V1RpcHandler::new(state) + .attest(AttestRequest { + report_data: vec![0; 65], + include_boottime_gpu_evidence: false, + }) + .await + .unwrap_err() + .to_string(); + assert!(err.contains("too long"), "{err}"); + } + + #[tokio::test] + async fn version_answers() { + let (_, state, _guard) = handler().await; + let response = V1RpcHandler::new(state) + .version(VersionRequest {}) + .await + .unwrap(); + assert!(!response.version.is_empty()); + } + + /// The external v1 surface attests the same app key the frozen + /// `GetAttestationForAppKey` commits to, but wraps it in an attestation, so + /// it answers on a platform with no TDX quote. That is the whole point of + /// the replacement. + #[tokio::test] + async fn external_attest_app_key_answers_without_a_tdx_quote() { + use ra_tls::attestation::{PlatformEvidence, VersionedAttestation}; + + let (state, _guard) = crate::rpc_service::tests::setup_test_state_with_platform(Some( + PlatformEvidence::SevSnp { + report: vec![0u8; 1184], + cert_chain: Vec::new(), + mr_config: String::new(), + }, + )) + .await; + + let response = ExternalV1RpcHandler::new(state) + .attest_app_key(AttestAppKeyRequest { + algorithm: "ed25519".to_string(), + }) + .await + .expect("v1 must attest an app key on a non-TDX platform"); + + let report_data = VersionedAttestation::from_bytes(&response.attestation) + .unwrap() + .into_v1() + .report_data() + .unwrap(); + assert_eq!( + b"dip1::ed25519-pk:5Pbre1Amf1hrp2V2bbfKlIfxpQb2pJAmrgmhxgVoG9s\0\0\0\0", + &report_data + ); + // This method attests a key, not the machine. + assert!(response.boottime_gpu_evidence.is_empty()); + } + + /// The external surface honours the app's `public_tcbinfo` choice; the + /// internal one has nobody to hide from. + #[tokio::test] + async fn the_external_surface_hides_documents_unless_the_app_opted_in() { + let (_, state, _guard) = handler().await; + assert!( + !state.config().app_compose.public_tcbinfo, + "the fixture must default to private for this test to mean anything" + ); + + let external = ExternalV1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + assert_eq!(external.app_compose, ""); + assert_eq!(external.vm_config, ""); + assert_eq!(external.key_provider_info, ""); + + // Identity and the measurement hashes stay visible, which is the line + // the unversioned `Worker.Info` drew too. + let internal = V1RpcHandler::new(state).info(InfoRequest {}).await.unwrap(); + assert_eq!(external.app_id, internal.app_id); + assert_eq!(external.instance_id, internal.instance_id); + assert_eq!(external.compose_hash, internal.compose_hash); + assert_eq!(external.os_image_hash, internal.os_image_hash); + assert_eq!(external.mr_aggregated, internal.mr_aggregated); + } + + /// An app that opted in gets the full response on the external surface. + #[tokio::test] + async fn the_external_surface_serves_documents_when_the_app_opted_in() { + let (state, _guard) = + crate::rpc_service::tests::setup_test_state_with_public_tcbinfo().await; + let external = ExternalV1RpcHandler::new(state.clone()) + .info(InfoRequest {}) + .await + .unwrap(); + assert_eq!(external.app_compose, state.config().app_compose.raw); + } + + /// An app that never opted into health gating is never polled, and anything + /// that asks anyway gets the answer the gateway would have assumed. + #[tokio::test] + async fn health_fails_open_for_an_app_that_did_not_opt_in() { + let (_, state, _guard) = handler().await; + let response = ExternalV1RpcHandler::new(state) + .health(HealthRequest {}) + .await + .unwrap(); + assert!(response.healthy); + assert!(response.unhealthy.is_empty()); + assert!(response.error.is_empty()); + } + + /// Route-level check without a live socket: the two generated dispatchers + /// own disjoint method tables, which is what makes mounting one at `/` and + /// the other at `/v1` a version selector rather than a name collision. + #[test] + fn the_two_surfaces_expose_different_method_sets() { + use dstack_guest_agent_rpc::dstack_guest_server::DstackGuestServer; + + let v0 = DstackGuestServer::::supported_methods(); + let v1 = DstackGuestV1Server::::supported_methods(); + + assert_eq!( + v1, + &[ + "IssueCert", + "GetKey", + "Attest", + "AttestGpu", + "Info", + "Version" + ], + "the v1 surface changed" + ); + + // The unversioned surface is closed at exactly v0.5.11. + assert_eq!( + v0, + &[ + "GetTlsKey", + "GetKey", + "GetQuote", + "Attest", + "EmitEvent", + "Info", + "Sign", + "Verify", + "Version" + ], + "the unversioned surface is frozen at the v0.5.11 method set" + ); + + // Everything v1 deliberately does not serve. `Sign` and `Verify` are + // pure computation over a key the caller can already fetch, `GetQuote` + // is the TDX-only channel `Attest` subsumes, and RTMR3 is + // system-owned. All four stay on the frozen surface. + for dropped in ["Sign", "Verify", "GetQuote", "EmitEvent"] { + assert!(!v1.contains(&dropped), "v1 must not serve {dropped}"); + } + + // v0 keeps the old name for what v1 calls `IssueCert`. + assert!(!v1.contains(&"GetTlsKey")); + + // Never-released `next` additions that now live only in v1, or nowhere. + assert!(!v0.contains(&"AttestGpu")); + assert!(v1.contains(&"AttestGpu")); + assert!(!v0.contains(&"GpuInfo") && !v1.contains(&"GpuInfo")); + } + + /// The external pair, checked the same way. + #[test] + fn the_two_external_surfaces_expose_different_method_sets() { + use dstack_guest_agent_rpc::v1::worker_v1_server::WorkerV1Server; + use dstack_guest_agent_rpc::worker_server::WorkerServer; + + let v0 = WorkerServer::::supported_methods(); + let v1 = WorkerV1Server::::supported_methods(); + + // Closed at v0.5.11: `AttestAppKey` and `Health` are post-0.5.11 and + // never released, so they live only on v1. + assert_eq!( + v0, + &["Info", "Version", "GetAttestationForAppKey"], + "the unversioned external surface is frozen at the v0.5.11 method set" + ); + assert_eq!(v1, &["Info", "Version", "AttestAppKey", "Health"]); + } +} diff --git a/dstack/guest-agent/src/rpc_service_v1/keys.rs b/dstack/guest-agent/src/rpc_service_v1/keys.rs new file mode 100644 index 000000000..f595176bd --- /dev/null +++ b/dstack/guest-agent/src/rpc_service_v1/keys.rs @@ -0,0 +1,434 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Key derivation and signature-chain claims for the v1 API. +//! +//! Everything here is normative wire behaviour: the byte strings this module +//! builds are what relying parties re-derive when they check a chain, and +//! `docs/guest-api-v1.md` is the specification they implement from. Changing a +//! constant or a field order here changes every key and every claim a deployed +//! agent produces. + +use anyhow::{anyhow, bail, Context, Result}; +use ed25519_dalek::SigningKey as Ed25519SigningKey; +use k256::ecdsa::SigningKey; +use ra_tls::kdf::derive_key; +use sha3::{Digest, Keccak256}; + +/// Context tag bound into every v1 key derivation. +pub(crate) const KEY_CONTEXT_TAG: &[u8] = b"dstack-guest-v1-key"; + +/// Context tag bound into every v1 signature-chain key claim. +/// +/// Distinct from [`KEY_CONTEXT_TAG`] so that no derivation input can ever be +/// read as a claim, or the other way round -- the two encodings are otherwise +/// built the same way and would share a prefix. +pub(crate) const CLAIM_CONTEXT_TAG: &[u8] = b"dstack-guest-v1-key-claim"; + +/// The key types the v1 API serves. +/// +/// A closed set, matched exhaustively: an algorithm name that is not one of +/// these is an error, never a default. v0 defaulted an empty string to +/// secp256k1 and accepted `k256` as an alias, so a caller could ask for +/// nothing in particular and get a key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Algorithm { + Secp256k1, + Ed25519, +} + +impl Algorithm { + pub(crate) fn parse(name: &str) -> Result { + match name { + "secp256k1" => Ok(Self::Secp256k1), + "ed25519" => Ok(Self::Ed25519), + "" => bail!("algorithm is required, use `secp256k1` or `ed25519`"), + other => bail!("unsupported algorithm `{other}`, use `secp256k1` or `ed25519`"), + } + } + + /// The canonical name, and the exact bytes bound into derivations and + /// claims. Never the string the caller sent. + pub(crate) fn name(self) -> &'static str { + match self { + Self::Secp256k1 => "secp256k1", + Self::Ed25519 => "ed25519", + } + } +} + +/// Append one length-prefixed field: a 4-byte big-endian length, then the +/// bytes. +/// +/// Prefixing rather than joining with a delimiter is the whole point. A domain +/// is an arbitrary caller-chosen string -- it may contain `:`, `/`, or NUL -- +/// so any delimiter it could also contain lets two different `(domain, +/// algorithm)` pairs encode to the same byte string and share a key. +fn push_field(out: &mut Vec, field: &[u8]) -> Result<()> { + let len = u32::try_from(field.len()).context("field is too long to encode")?; + out.extend_from_slice(&len.to_be_bytes()); + out.extend_from_slice(field); + Ok(()) +} + +/// The HKDF `info` for a v1 application key. +/// +/// `LP(tag) || LP(algorithm) || LP(domain)`, where `LP(x)` is `len(x)` as a +/// 4-byte big-endian integer followed by `x`. +pub(crate) fn key_derivation_info(domain: &str, algorithm: Algorithm) -> Result> { + let mut info = Vec::new(); + push_field(&mut info, KEY_CONTEXT_TAG)?; + push_field(&mut info, algorithm.name().as_bytes())?; + push_field(&mut info, domain.as_bytes())?; + Ok(info) +} + +/// The claim the app root key signs to vouch for a derived public key. +/// +/// `LP(tag) || LP(algorithm) || LP(domain) || LP(public_key)`, with the public +/// key as raw bytes. +/// +/// Raw bytes, and a length prefix in front of them, are what makes this +/// unforgeable through the v0 surface. v0's claim is +/// `keccak256("{purpose}:{hex(pubkey)}")` over a caller-chosen `purpose`, so +/// the app root key can be made to sign nearly any ASCII string that ends in +/// `:` followed by lowercase hex. This encoding ends in `LP(public_key)`, +/// whose four length bytes are `00 00 00 21` for secp256k1 and `00 00 00 20` +/// for ed25519 and sit inside the region a v0 preimage requires to be +/// hex-only. `0x00` is not a hex character, so no `purpose` reproduces this +/// byte string -- the exclusion is structural, not probabilistic. +pub(crate) fn key_claim(algorithm: Algorithm, domain: &str, public_key: &[u8]) -> Result> { + let mut claim = Vec::new(); + push_field(&mut claim, CLAIM_CONTEXT_TAG)?; + push_field(&mut claim, algorithm.name().as_bytes())?; + push_field(&mut claim, domain.as_bytes())?; + push_field(&mut claim, public_key)?; + Ok(claim) +} + +/// An application key derived for one `(domain, algorithm)` pair. +pub(crate) struct AppKey { + algorithm: Algorithm, + domain: String, + secret: [u8; 32], + public_key: Vec, +} + +impl AppKey { + /// Derive the key for `(domain, algorithm)` from the app root key. + /// + /// HKDF-SHA256 over the app root secp256k1 key, with the `info` from + /// [`key_derivation_info`]. Same primitive and same salt as v0; what + /// changed is that the algorithm and a version tag are now inputs, so the + /// two curves no longer share one secret and a v1 domain is not a v0 path. + /// + /// Flat, not hierarchical: `a/b` is an opaque domain string like any + /// other, unrelated to `a`, and no key here derives another. + pub(crate) fn derive(app_root_key: &[u8], domain: &str, algorithm: Algorithm) -> Result { + let info = key_derivation_info(domain, algorithm)?; + let derived = derive_key(app_root_key, &[&info], 32) + .map_err(|_| anyhow!("failed to derive the application key"))?; + let secret: [u8; 32] = derived + .as_slice() + .try_into() + .map_err(|_| anyhow!("derived key has the wrong length"))?; + let public_key = match algorithm { + // Rejects a scalar that is zero or at least the group order. That + // is a ~2^-128 event for one domain and the caller can just pick + // another one, so failing is better than folding the scalar into + // range and quietly landing two domains on one key. + Algorithm::Secp256k1 => SigningKey::from_slice(&secret) + .context("derived secp256k1 key is not a valid scalar")? + .verifying_key() + .to_sec1_bytes() + .to_vec(), + Algorithm::Ed25519 => Ed25519SigningKey::from_bytes(&secret) + .verifying_key() + .to_bytes() + .to_vec(), + }; + Ok(Self { + algorithm, + domain: domain.to_string(), + secret, + public_key, + }) + } + + /// The raw 32-byte private key. + pub(crate) fn secret(&self) -> Vec { + self.secret.to_vec() + } + + /// SEC1 compressed for secp256k1, 32 raw bytes for ed25519. + pub(crate) fn public_key(&self) -> Vec { + self.public_key.clone() + } + + /// The app root key's signature over this key's claim: the first link of + /// the signature chain. + /// + /// 65 bytes, `r || s || recovery_id`, over `keccak256` of the claim. The + /// recovery byte is kept so a relying party can recover the app root + /// public key from the link alone and compare it against what the second + /// link attests. + pub(crate) fn claim_signature(&self, app_root_key: &[u8]) -> Result> { + let claim = key_claim(self.algorithm, &self.domain, &self.public_key)?; + let app_signing_key = SigningKey::from_slice(app_root_key) + .context("failed to parse the app root k256 key")?; + let (signature, recovery_id) = app_signing_key + .sign_digest_recoverable(Keccak256::new_with_prefix(&claim)) + .context("failed to sign the key claim")?; + let mut link = signature.to_vec(); + link.push(recovery_id.to_byte()); + Ok(link) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The app root key the committed vectors were generated from. Same value + /// the handler tests use, so a vector can be reproduced end to end. + const TEST_APP_ROOT_KEY: [u8; 32] = [ + 0x1A, 0x2B, 0x3C, 0x4D, 0x5E, 0x6F, 0x7A, 0x8B, 0x9C, 0x0D, 0x1E, 0x2F, 0x3A, 0x4B, 0x5C, + 0x6D, 0x7E, 0x8F, 0x9A, 0x0B, 0x1C, 0x2D, 0x3E, 0x4F, 0x5A, 0x6B, 0x7C, 0x8D, 0x9E, 0x0F, + 0x1A, 0x2B, + ]; + + fn derive(domain: &str, algorithm: Algorithm) -> AppKey { + AppKey::derive(&TEST_APP_ROOT_KEY, domain, algorithm).unwrap() + } + + #[test] + fn rejects_an_empty_or_unknown_algorithm() { + assert_eq!( + Algorithm::parse("").unwrap_err().to_string(), + "algorithm is required, use `secp256k1` or `ed25519`" + ); + assert_eq!( + Algorithm::parse("k256").unwrap_err().to_string(), + "unsupported algorithm `k256`, use `secp256k1` or `ed25519`" + ); + assert!(Algorithm::parse("secp256k1_prehashed").is_err()); + assert!(Algorithm::parse("rsa").is_err()); + } + + #[test] + fn encodes_derivation_info_with_length_prefixes() { + let info = key_derivation_info("wallet", Algorithm::Secp256k1).unwrap(); + let expected = [ + &19u32.to_be_bytes()[..], + b"dstack-guest-v1-key", + &9u32.to_be_bytes()[..], + b"secp256k1", + &6u32.to_be_bytes()[..], + b"wallet", + ] + .concat(); + assert_eq!(info, expected); + } + + /// A delimiter-joined encoding would collide here; a length-prefixed one + /// cannot. A domain carrying the delimiter is the shape that breaks + /// `join(":")`. + #[test] + fn no_two_domains_encode_the_same_way() { + let a = key_derivation_info("a\u{0}b", Algorithm::Ed25519).unwrap(); + let b = key_derivation_info("a", Algorithm::Ed25519).unwrap(); + let c = key_derivation_info("ab", Algorithm::Ed25519).unwrap(); + assert_ne!(a, b); + assert_ne!(a, c); + assert_ne!(b, c); + } + + /// Committed vectors, also published in `docs/guest-api-v1.md` so a + /// non-Rust implementation can check itself against them. + /// + /// These bytes are the v1 keys of every deployment whose app root key is + /// `TEST_APP_ROOT_KEY`. A diff here is a change to deployed key material, + /// not a fixture update: fix the derivation, do not update the vector. + #[test] + fn derives_the_committed_key_vectors() { + let vectors = [ + ( + "", + Algorithm::Secp256k1, + "463e877bc7322c1c09e567844b3101e88f353bfb33177c41cb13832cb67eef1c", + "03c45e036d19662802e628d9a712c07a9d9d64bee28e1754a72d75010860a789c2", + ), + ( + "", + Algorithm::Ed25519, + "a41f6458de9d11a43f79640b6ab2c62d02aceda7b16c5f159dc7ad69621c7eb0", + "7a5740fa9ab4791a232cc1fc7e73d5ff47ad41be2589a71b05f128dee4223f08", + ), + ( + "wallet", + Algorithm::Secp256k1, + "c2b47271c2956c020eb471f3d6ec9a08bac4ce72d158078592c3bfd9db67808c", + "0375b11b6fabbe6e18b9bac26b082070dea76487ce512323870bc784a28ec5404b", + ), + ( + "wallet", + Algorithm::Ed25519, + "fce7a47f848fc9f7a799a34e061f4d07d7e8ef3adda98d8fc36a5de3f5146cdf", + "5e8076b492634770e9b12dc8b136c9f5e3e8a86adc657040ba7a320296dcf6b0", + ), + ( + "a/b/c", + Algorithm::Secp256k1, + "448f92c86abd72c1c35b7ff75b5c9971114694e7825518e5ed0d2101711527cd", + "03011845be1d30004c148ae49ef3a82585094f856a7c9bc3ed06edf959537a4eac", + ), + // A domain carrying NUL and `:`, the two characters a + // delimiter-joined encoding would choke on. + ( + "k\u{0}:ey", + Algorithm::Ed25519, + "3cfcf09543e94d23c87aec1e5774ca3803feb3dd8ee298507650409b20ebfdde", + "9ba2d94591b97db4f089fc187cfcb09a9cd55bdd553e066d80cc8cb2977bd841", + ), + ]; + for (domain, algorithm, expected_secret, expected_public) in vectors { + let key = derive(domain, algorithm); + let name = algorithm.name(); + assert_eq!( + hex::encode(key.secret()), + expected_secret, + "v1 key vector changed for ({domain:?}, {name})" + ); + assert_eq!( + hex::encode(key.public_key()), + expected_public, + "v1 public key vector changed for ({domain:?}, {name})" + ); + } + } + + /// The claim signature is deterministic: RFC 6979 fixes `k` from the key + /// and the digest, so a committed vector pins the whole chain-link + /// encoding, recovery byte included. + #[test] + fn produces_the_committed_claim_signature_vector() { + let key = derive("wallet", Algorithm::Secp256k1); + assert_eq!( + hex::encode(key.claim_signature(&TEST_APP_ROOT_KEY).unwrap()), + "96726db35263cb2a7067fc5ebb0d06621f7cab2d0f0aa15a83858dbf9ff7f12c\ + 6e1c5640223fcca1a03ba5ccf09d353609db6481f9563add16e16a8ad8544c29\ + 00" + ); + } + + #[test] + fn the_two_algorithms_never_share_key_material() { + for domain in ["", "wallet", "a/b/c", "\u{0}"] { + assert_ne!( + derive(domain, Algorithm::Secp256k1).secret(), + derive(domain, Algorithm::Ed25519).secret(), + "cross-algorithm key reuse at {domain:?}" + ); + } + } + + /// v0 derived `derive_key(app_root, [path], 32)` and handed the same 32 + /// bytes to both curves. v1 must not land on those bytes when the domain + /// string equals the old path. + #[test] + fn v1_keys_differ_from_v0_keys_for_the_same_name() { + for name in ["", "wallet", "vms"] { + let v0 = derive_key(&TEST_APP_ROOT_KEY, &[name.as_bytes()], 32).unwrap(); + assert_ne!(derive(name, Algorithm::Secp256k1).secret(), v0); + assert_ne!(derive(name, Algorithm::Ed25519).secret(), v0); + } + } + + #[test] + fn encodes_the_claim_with_the_raw_public_key() { + let key = derive("wallet", Algorithm::Secp256k1); + let public_key = key.public_key(); + assert_eq!(public_key.len(), 33, "expected a compressed SEC1 point"); + let claim = key_claim(Algorithm::Secp256k1, "wallet", &public_key).unwrap(); + let expected = [ + &25u32.to_be_bytes()[..], + b"dstack-guest-v1-key-claim", + &9u32.to_be_bytes()[..], + b"secp256k1", + &6u32.to_be_bytes()[..], + b"wallet", + &33u32.to_be_bytes()[..], + &public_key, + ] + .concat(); + assert_eq!(claim, expected); + } + + /// The forgery a malicious app would attempt: v0's `purpose` is + /// caller-chosen, so it picks one that reproduces the v1 claim's prefix + /// and lets v0 append `":" + hex(pubkey)` itself. + #[test] + fn a_v0_claim_cannot_be_crafted_into_a_v1_claim() { + let key = derive("wallet", Algorithm::Secp256k1); + let public_key = key.public_key(); + let v1_claim = key_claim(Algorithm::Secp256k1, "wallet", &public_key).unwrap(); + + // Best case for the attacker: the v1 claim minus exactly the suffix v0 + // appends on its own, used verbatim as `purpose`. + let hex_pubkey = hex::encode(&public_key); + let appended = format!(":{hex_pubkey}"); + let purpose_len = v1_claim.len().saturating_sub(appended.len()); + let purpose = String::from_utf8_lossy(&v1_claim[..purpose_len]).into_owned(); + let v0_claim = format!("{purpose}:{hex_pubkey}").into_bytes(); + + assert_ne!( + v0_claim, v1_claim, + "a v0 purpose reproduced the v1 claim byte string" + ); + assert_ne!( + Keccak256::digest(&v0_claim), + Keccak256::digest(&v1_claim), + "a v0 claim collided with a v1 claim" + ); + + // And the structural reason, so a future encoding change cannot make + // the assertion above pass by accident: a v0 preimage ends in `:` + // followed by lowercase hex only, while the v1 claim's last 37 bytes + // start with the public key's `00 00 00 21` length prefix. + let tail = &v1_claim[v1_claim.len() - 37..]; + assert_eq!(&tail[..4], &33u32.to_be_bytes()); + assert!( + tail.iter().any(|b| !b.is_ascii_hexdigit()), + "the v1 claim tail is entirely hex, which v0 could reproduce" + ); + } + + #[test] + fn the_two_context_tags_are_not_prefixes_of_one_another_once_encoded() { + let derivation = key_derivation_info("p", Algorithm::Ed25519).unwrap(); + let claim = key_claim(Algorithm::Ed25519, "p", &[0u8; 32]).unwrap(); + assert!(!claim.starts_with(&derivation)); + assert!(!derivation.starts_with(&claim)); + } + + #[test] + fn the_claim_signature_verifies_under_the_app_root_key() { + use k256::ecdsa::{RecoveryId, Signature as K256Signature, VerifyingKey}; + + let key = derive("wallet", Algorithm::Secp256k1); + let link = key.claim_signature(&TEST_APP_ROOT_KEY).unwrap(); + assert_eq!(link.len(), 65); + + let claim = key_claim(Algorithm::Secp256k1, "wallet", &key.public_key()).unwrap(); + let digest = Keccak256::new_with_prefix(&claim); + let signature = K256Signature::from_slice(&link[..64]).unwrap(); + let recovery_id = RecoveryId::from_byte(link[64]).unwrap(); + + let recovered = VerifyingKey::recover_from_digest(digest, &signature, recovery_id).unwrap(); + let app_root = *SigningKey::from_slice(&TEST_APP_ROOT_KEY) + .unwrap() + .verifying_key(); + assert_eq!(recovered, app_root); + } +} diff --git a/dstack/guest-agent/src/server.rs b/dstack/guest-agent/src/server.rs index 53a0690ca..8551e85bc 100644 --- a/dstack/guest-agent/src/server.rs +++ b/dstack/guest-agent/src/server.rs @@ -8,6 +8,7 @@ use crate::config::BindAddr; use crate::guest_api_service::GuestApiHandler; use crate::http_routes; use crate::rpc_service::{AppState, ExternalRpcHandler, InternalRpcHandler, InternalRpcHandlerV0}; +use crate::rpc_service_v1::{ExternalV1RpcHandler, V1RpcHandler}; use crate::socket_activation::{ActivatedSockets, ActivatedUnixListener}; use anyhow::{anyhow, Context, Result}; use ra_rpc::rocket_helper::UnixPeerCredListener; @@ -86,8 +87,13 @@ async fn run_internal( activated_socket: Option, sock_ready_tx: oneshot::Sender<()>, ) -> Result<()> { + // Two surfaces, one socket, selected by URL path alone. `/` is the frozen + // unversioned API v0.5.x clients speak; `/v1` is `dstack.guest.v1`. There + // is no header negotiation and no default-version redirect, so a caller's + // URL is the whole record of which contract it asked for. let rocket = rocket::custom(figment) .mount("/", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) + .mount("/v1", ra_rpc::prpc_routes!(AppState, V1RpcHandler)) .manage(state); let ignite = rocket .ignite() @@ -135,10 +141,16 @@ async fn run_internal( async fn run_external(state: AppState, figment: Figment) -> Result<()> { let rocket = rocket::custom(figment) .mount("/", http_routes::external_routes(state.config())) + // Same two-surface split as the internal socket: `/prpc` is the + // unversioned Worker, closed at v0.5.11, and `/prpc/v1` is `WorkerV1`. .mount( "/prpc", ra_rpc::prpc_routes!(AppState, ExternalRpcHandler, trim: "Worker."), ) + .mount( + "/prpc/v1", + ra_rpc::prpc_routes!(AppState, ExternalV1RpcHandler), + ) .attach(AdHoc::on_response("Add app version header", |_req, res| { Box::pin(async move { res.set_raw_header("X-App-Version", app_version()); diff --git a/dstack/kms/src/main_service/upgrade_authority.rs b/dstack/kms/src/main_service/upgrade_authority.rs index c315e0128..9340dd650 100644 --- a/dstack/kms/src/main_service/upgrade_authority.rs +++ b/dstack/kms/src/main_service/upgrade_authority.rs @@ -5,7 +5,9 @@ use super::build_boot_info_for_attestation; use crate::config::{AuthApi, KmsConfig}; use anyhow::{bail, Context, Result}; -use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, AttestArgs, AttestResponse}; +use dstack_guest_agent_rpc::{ + dstack_guest_client::DstackGuestClient, AttestResponse, RawQuoteArgs, +}; use http_client::prpc::PrpcClient; use ra_tls::attestation::{AttestationVerifier, VerifiedAttestation, VersionedAttestation}; use serde::de::DeserializeOwned; @@ -180,12 +182,7 @@ pub(crate) fn dstack_client() -> DstackGuestClient { } pub(crate) async fn app_attest(report_data: Vec) -> Result { - dstack_client() - .attest(AttestArgs { - report_data, - include_boottime_gpu_evidence: false, - }) - .await + dstack_client().attest(RawQuoteArgs { report_data }).await } pub(crate) fn pad64(hash: [u8; 32]) -> Vec { From 3f8a5827ed539b926b1495e113fe9226f8dda748 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 02:50:32 -0700 Subject: [PATCH 3/7] docs: specify the guest agent v1 API --- README.md | 1 + docs/app-health-checks.md | 2 +- docs/guest-api-v1.md | 593 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 595 insertions(+), 1 deletion(-) create mode 100644 docs/guest-api-v1.md diff --git a/README.md b/README.md index 0419cb498..8444cf88e 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ Apps communicate with the guest agent via HTTP over `/var/run/dstack.sock`. Use - [Gateway](./docs/dstack-gateway.md) - Gateway configuration **Reference** +- [Guest Agent API v1](./docs/guest-api-v1.md) - Key derivation, signature chains, and the versioned guest API - [App Compose Format](./docs/normalized-app-compose.md) - Compose file specification - [Intel TDX Attestation](./docs/attestation-tdx.md) - Measurement and runtime-event verification - [Native TEE Interfaces](./docs/native-tee-interfaces.md) - Advanced compatibility with Linux TEE devices and configfs-tsm diff --git a/docs/app-health-checks.md b/docs/app-health-checks.md index 0f574c299..f3e5ae8ea 100644 --- a/docs/app-health-checks.md +++ b/docs/app-health-checks.md @@ -46,7 +46,7 @@ older guest chokes on. ## Where the verdict comes from The guest agent recomputes a verdict every 5 seconds and caches it; the gateway -polls `Worker.Health`, which only reads that cache. The two cadences are +polls `WorkerV1.Health` (at `/prpc/v1`), which only reads that cache. The two cadences are independent on purpose — a fleet of gateway nodes polling the same instance must not multiply into that many container-runtime queries inside the CVM, and the RPC is served on the CVM's publicly reachable listener. diff --git a/docs/guest-api-v1.md b/docs/guest-api-v1.md new file mode 100644 index 000000000..2956087b9 --- /dev/null +++ b/docs/guest-api-v1.md @@ -0,0 +1,593 @@ +# dstack Guest Agent API v1 + +This is the normative specification of `dstack.guest.v1`, the versioned guest +agent API introduced in dstack 0.6.0. It defines the URL scheme, the key +derivation function, the signature chain encoding, and the steps a relying party +follows to verify what the agent produces. + +Read it as the contract. The proto file +(`dstack/guest-agent/rpc/proto/agent_rpc_v1.proto`) describes the message shapes; +this document pins the bytes. Where an implementation and this document disagree, +one of them is a bug. + +## What v1 is for + +v1 exists because the unversioned surface cannot be fixed without breaking the +v0.5.x clients that depend on it. That surface is closed at exactly the v0.5.11 +shape. v1 is where the corrections live. + +The guest agent is not a general-purpose crypto service. It holds two things no +caller can obtain elsewhere: the application root key, and the platform's ability +to attest. v1 serves those two things and nothing else. + +That is the organising principle behind the method set. `Sign` and `Verify` are +absent because they are pure computation over material the caller already has. +Any process that can reach this socket can ask `GetKey` for the private key +itself, so a server-side `Sign` grants no capability, adds an IPC round trip, and +adds an entry point to audit. Verification does not even need a key. + +Four things changed for the operations that remain. + +**Keys are domain-separated.** v0 derived a key from a path alone and handed the +same 32 bytes to both secp256k1 and ed25519. v1 binds the algorithm and a +versioned context tag into the derivation, so the two curves never share a secret +and a v1 key is never a v0 key. + +**A key is named by one field.** v0 had `path` and `purpose`, of which only `path` +reached the KDF while `purpose` was echoed into the chain claim. v1 merges them +into `domain`, and both `domain` and `algorithm` affect the derived key. + +**The chain claim is unambiguous.** v0's claim is a `:`-joined string over a +caller-chosen `purpose`, which lets an application steer what the app root key +signs. v1's claim is length-prefixed and binds raw public key bytes. + +**Names say what things are.** `GetTlsKey` became `IssueCert`, because the +operation is certificate issuance. `Info` no longer nests documents inside a JSON +blob called `tcb_info`. + +## Transport and versioning + +Every surface the agent serves is now a closed v0 fossil plus a v1. The +unversioned surfaces are exactly v0.5.11 and never change again; new capability +arrives only in `dstack.guest.v1`. + +| Listener | Service | Mount | Example path | +|---|---|---|---| +| Internal socket | `DstackGuestV1` | `/v1` | `/v1/GetKey` | +| Internal socket | `DstackGuest`, closed | `/` | `/GetKey` | +| Internal socket | `Tappd`, closed | `/prpc/` | `/prpc/Tappd.TdxQuote` | +| External | `WorkerV1` | `/prpc/v1` | `/prpc/v1/Health` | +| External | `Worker`, closed | `/prpc` | `/prpc/Worker.Info` | + +The internal socket is `/var/run/dstack.sock`, reachable only by the application +itself. The external listener is reachable by anyone who can route to the CVM. +That difference is the whole reason there are two services rather than one +mounted twice: `DstackGuestV1` hands out key material, and `WorkerV1` never does. + +Version selection is by URL path and nothing else. There is no header +negotiation, no `Accept-Version`, and no default-version redirect, so a request +URL is the complete record of which contract the caller asked for. An agent that +predates v1 answers `/v1/...` with HTTP 400, the same way it answers any unknown +method. + +Both surfaces run over the same prpc transport. A `POST` carrying +`Content-Type: application/json` takes a JSON body and returns JSON; any other +content type takes a protobuf-encoded request message and returns a +protobuf-encoded response. A `GET` takes its fields as query parameters and +returns JSON. + +Every handler failure returns HTTP 400 with the error text in the body. prpc uses +the same status for an unknown method, so the message text is the only thing that +distinguishes "this agent does not have that method" from "that call failed". + +## The internal surface + +`DstackGuestV1` has six methods. + +| Method | Purpose | +|---|---| +| `IssueCert` | Issue a certificate, with a freshly generated key | +| `GetKey` | Derive an application key and return it with its signature chain | +| `Attest` | Produce a versioned attestation over caller-supplied report data | +| `AttestGpu` | Collect GPU evidence now, against a caller-chosen nonce | +| `Info` | Return application identity and configuration | +| `Version` | Return the agent version | + +Five v0 methods are deliberately absent. + +`Sign` and `Verify` are absent for the reason above: neither needs the TEE. +Applications sign locally with a standard crypto library, using the key `GetKey` +returns, and verify locally following this document. A v0 client may keep calling +the unversioned `Verify` for single-signature checks; the v0 `Sign` RPC's +per-algorithm signing modes are documented on that surface, not here. + +`GetQuote` is absent because `Attest` subsumes it. `GetQuote` answers on Intel TDX +and nowhere else, and the `VersionedAttestation` that `Attest` returns already +carries the TDX quote and the event log. See +[Extracting a quote from an attestation](#extracting-a-quote-from-an-attestation). + +`GpuInfo` is absent from v1, and has also been removed from the unversioned +surface. It never appeared in a release, and +`Attest` with `include_boottime_gpu_evidence` returns the same bytes. + +`EmitEvent` is absent because runtime RTMR3 events became system-owned in 0.6.0. +An application binds its data through `report_data` instead. + +### Naming conventions + +Every request message is `Request` and every response is +`Response`, including for methods that take no arguments. An empty message +can gain a field later; `google.protobuf.Empty` cannot. + +A field's encoding lives in its doc comment, not in its name. Fields carrying JSON +documents say so and name who owns the schema. + +## Certificate issuance + +`IssueCert` is certificate issuance. The agent generates a key, builds a CSR, +signs the CSR with that key, and relays it to the KMS `SignCert` RPC, or to the +local CA when the application runs without a KMS. It returns the chain the signer +produced. + +v0 called this `GetTlsKey`, which named the by-product rather than the request. +The private key is incidental: it is freshly generated on every call, no request +field feeds it, and two identical requests produce two unrelated keys. `GetKey` is +the method that derives a stable, attestable key. + +`not_before` must be earlier than `not_after` when both are set; otherwise the +call fails. + +This first cut serves only the integrated one-step mode, where the agent holds the +key. A mode that signs a caller-supplied CSR or public key, so the private key +never leaves the caller, is a plausible extension. It would arrive as added fields +or a sibling method, never as a change to what these fields mean. + +## Key derivation + +### Inputs + +A v1 application key is named by exactly two values. + +`domain` is a caller-chosen domain-separation string. It is not a DNS name; the +certificate fields on `IssueCertRequest` are the ones that take those. It may be +any byte string a proto3 `string` can carry, including one containing `:`, `/`, or +NUL. + +`algorithm` is `secp256k1` or `ed25519`. There is no default and no alias. An +empty string is an error, and so is any other value, including `k256` and +`secp256k1_prehashed`. + +Derivation is **flat**. Two domains yield unrelated keys. `a/b` is an opaque +string that happens to contain a slash, not a child of `a`, and no key derived +here can be used to derive another. There is no BIP-32-style hierarchy. The old +`path` name invited that reading, which is part of why it is gone. + +### Length-prefixed encoding + +Every construction below builds its input from length-prefixed fields. Write +`LP(x)` for: + +```text +LP(x) = uint32_be(len(x)) || x +``` + +`len(x)` is the byte length, encoded big-endian in exactly four bytes. Encoding +fails if a field is longer than 2^32 - 1 bytes. + +The prefix is what makes the encoding injective. A domain is arbitrary +caller-chosen bytes, so any delimiter it could also contain would let two +different `(domain, algorithm)` pairs encode to one byte string and share a key. +Joining with `:` or `/` is not sufficient here and is not what v1 does. + +### The KDF + +```text +salt = "RATLS" (5 bytes, ASCII, unchanged from v0) +IKM = app root secp256k1 private key (32 bytes, `k256_key` from .appkeys.json) +info = LP("dstack-guest-v1-key") || LP(algorithm) || LP(domain) +L = 32 + +key = HKDF-SHA256(salt, IKM, info, L) (RFC 5869: extract, then expand) +``` + +`algorithm` is bound as its canonical name, `secp256k1` or `ed25519`, never as the +string the caller sent. + +The primitive family is unchanged from v0, which used the same HKDF-SHA256 and the +same salt. What changed is the `info`: v0 passed the path alone, so the algorithm +did not participate and one 32-byte secret served both curves. + +The 32 output bytes are used directly: + +- **secp256k1**: the big-endian private scalar. If it is zero or at least the + group order, the call fails. That is a ~2^-128 event for a given domain, and the + caller can choose another one. Folding the scalar into range would silently land + two domains on one key. +- **ed25519**: the RFC 8032 seed, from which the key expands as usual. + +Because v1 and v0 share the HKDF salt, a caller that supplies a crafted `path` to +the unversioned `GetKey` can reproduce a v1 key. That is not a boundary. Both +derivations serve the same single-tenant application from the same root key, and +that application can call either surface. The domain separation here prevents +accidental reuse across algorithms and versions; it does not isolate a caller from +itself, and nothing in dstack's threat model asks it to. + +### Public key encoding + +| Algorithm | Encoding | Length | +|---|---|---| +| `secp256k1` | SEC1 compressed point, `0x02`/`0x03` prefix | 33 bytes | +| `ed25519` | RFC 8032 raw public key | 32 bytes | + +These are the exact bytes returned in `public_key` and the exact bytes bound into +the chain claim. A relying party never has to re-derive the public key from the +private key to check the chain. + +### Test vectors + +Generated with an app root key of + +```text +1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b +``` + +| domain | algorithm | private key | public key | +|---|---|---|---| +| `""` | secp256k1 | `463e877bc7322c1c09e567844b3101e88f353bfb33177c41cb13832cb67eef1c` | `03c45e036d19662802e628d9a712c07a9d9d64bee28e1754a72d75010860a789c2` | +| `""` | ed25519 | `a41f6458de9d11a43f79640b6ab2c62d02aceda7b16c5f159dc7ad69621c7eb0` | `7a5740fa9ab4791a232cc1fc7e73d5ff47ad41be2589a71b05f128dee4223f08` | +| `wallet` | secp256k1 | `c2b47271c2956c020eb471f3d6ec9a08bac4ce72d158078592c3bfd9db67808c` | `0375b11b6fabbe6e18b9bac26b082070dea76487ce512323870bc784a28ec5404b` | +| `wallet` | ed25519 | `fce7a47f848fc9f7a799a34e061f4d07d7e8ef3adda98d8fc36a5de3f5146cdf` | `5e8076b492634770e9b12dc8b136c9f5e3e8a86adc657040ba7a320296dcf6b0` | +| `a/b/c` | secp256k1 | `448f92c86abd72c1c35b7ff75b5c9971114694e7825518e5ed0d2101711527cd` | `03011845be1d30004c148ae49ef3a82585094f856a7c9bc3ed06edf959537a4eac` | +| `k\0:ey` | ed25519 | `3cfcf09543e94d23c87aec1e5774ca3803feb3dd8ee298507650409b20ebfdde` | `9ba2d94591b97db4f089fc187cfcb09a9cd55bdd553e066d80cc8cb2977bd841` | + +The last row's domain is the five bytes `6b 00 3a 65 79`. It is there because a +delimiter-joined encoding would mishandle it. + +The same vectors are asserted in +`dstack/guest-agent/src/rpc_service_v1/keys.rs`. They describe deployed key +material, so a diff against them is a bug in the derivation, not a stale fixture. + +## The signature chain + +`GetKey` returns a two-element `signature_chain` alongside the key. + +```text +[0] app root key signs the v1 key claim (specified here) +[1] KMS root key signs the app root public key (unchanged from v0) +``` + +Link 1 is produced by the KMS, outside the agent, and both API surfaces pass the +same bytes through. Only link 0 is new. + +### Link 0: the key claim + +```text +claim = LP("dstack-guest-v1-key-claim") + || LP(algorithm) + || LP(domain) + || LP(public_key) + +digest = keccak256(claim) +link0 = r || s || v (65 bytes) +``` + +`link0` is a recoverable ECDSA secp256k1 signature over `digest` by the app root +key. `r` and `s` are 32-byte big-endian integers, low-S normalised, and `v` is the +one-byte recovery id in `0..=3`. `public_key` is the raw derived public key from +the table above, not a hex string. + +A worked claim, for `domain = "wallet"` and `algorithm = "secp256k1"`: + +```text +00 00 00 19 "dstack-guest-v1-key-claim" 25 bytes +00 00 00 09 "secp256k1" 9 bytes +00 00 00 06 "wallet" 6 bytes +00 00 00 21 03 75 b1 1b ... 04 4b 33 bytes +``` + +With the app root key from the test vector table, `link0` is + +```text +96726db35263cb2a7067fc5ebb0d06621f7cab2d0f0aa15a83858dbf9ff7f12c +6e1c5640223fcca1a03ba5ccf09d353609db6481f9563add16e16a8ad8544c29 +00 +``` + +The signature is deterministic (RFC 6979), so this vector pins the whole encoding +including the recovery byte. + +#### Why this cannot be forged through v0 + +v0's claim is `keccak256("{purpose}:{hex(pubkey)}")`, and `purpose` is an +arbitrary caller-supplied string. A malicious application can therefore make the +app root key sign nearly any ASCII byte string that ends in `:` followed by +lowercase hex. If a v1 claim were reachable that way, the v1 chain would be worth +nothing. + +It is not reachable, and the reason is structural rather than probabilistic. + +Every v0 preimage ends with `:` followed by the hex encoding of a public key, so +its last 64 bytes (ed25519) or 66 bytes (secp256k1) are all lowercase hex +characters. A v1 claim ends with `LP(public_key)`, whose four length bytes are +`00 00 00 21` for secp256k1 and `00 00 00 20` for ed25519, sitting 37 or 36 bytes +from the end. That is inside the region a v0 preimage requires to be hex-only, and +`0x00` is not a hex character. No choice of `purpose` reproduces a v1 claim byte +string, and since the two byte strings can never be equal, matching their keccak +digests would require a preimage attack on keccak256. + +The regression test is `a_v0_claim_cannot_be_crafted_into_a_v1_claim` in +`dstack/guest-agent/src/rpc_service_v1/keys.rs`. It builds the strongest available +forgery, a `purpose` set to the v1 claim minus exactly the suffix v0 appends on +its own, then asserts both that the byte strings differ and that the structural +property holds. + +The two context tags are also distinct, so a derivation input can never be read as +a claim: `dstack-guest-v1-key` for the KDF, `dstack-guest-v1-key-claim` for the +claim. Because both are length-prefixed, neither encoding is a prefix of the other. + +### Link 1: the KMS attestation + +Unchanged from v0 and reproduced here so this document is self-contained. + +```text +message = "dstack-kms-issued" || ":" || app_id || sec1_compressed(app_root_pubkey) +digest = keccak256(message) +link1 = r || s || v (65 bytes) +``` + +`link1` is a recoverable ECDSA secp256k1 signature over `digest` by the KMS root +key. `app_id` is the raw app id bytes, and the app root public key is SEC1 +compressed, 33 bytes. + +### Verifying a chain + +A relying party holds a `public_key`, a `signature_chain`, the `(domain, +algorithm)` the key was requested under, and the `app_id`. It performs the +following. + +1. **Anchor.** Obtain the KMS root public key from a source you trust + independently of the agent being checked: the `DstackKms` contract's + `kmsInfo().k256Pubkey`, or a value pinned out of band. This step carries the + security of everything below it. An attacker who can answer your query for the + anchor can also mint a self-consistent chain, so reading the anchor from the + KMS you are checking proves nothing. + +2. **Rebuild the claim.** Compute + `claim = LP("dstack-guest-v1-key-claim") || LP(algorithm) || LP(domain) || LP(public_key)` + using the canonical algorithm name and the raw public key bytes, then + `digest0 = keccak256(claim)`. + +3. **Recover the app root key.** Split `signature_chain[0]` into `r`, `s`, `v` and + recover the secp256k1 public key from `(digest0, r, s, v)`. Reject a + non-canonical high-S `s`. Call the result `app_root_pubkey`, SEC1 compressed. + +4. **Rebuild the KMS message.** Compute + `digest1 = keccak256("dstack-kms-issued" || ":" || app_id || app_root_pubkey)`. + +5. **Check link 1.** Verify `signature_chain[1]` over `digest1` against the anchor + from step 1, either by recovering and comparing to the anchor or by verifying + `(r, s)` against it directly. Reject high-S here too. + +6. **Bind the application.** Confirm the `app_id` you used in step 4 is the + application you meant to talk to. The chain proves that the KMS issued this app + root key to *some* application; only this step ties it to yours. + +To also check a payload signature an application produced with a key from +`GetKey`, verify that signature against `public_key` under whatever scheme the +application used, then run steps 2 through 6 to establish that `public_key` is what +it claims to be. The two halves are independent: a valid payload signature under an +unverified public key says nothing. + +Step 3 recovers the app root key rather than requiring it as an input, which is why +link 0 carries a recovery byte. A verifier that already knows the expected app root +public key may verify `(r, s)` against it directly and compare instead. + +## The external surface + +`WorkerV1` has four methods, served at `/prpc/v1`. + +| Method | Purpose | +|---|---| +| `Info` | Application identity and configuration, subject to `public_tcbinfo` | +| `Version` | The agent version | +| `AttestAppKey` | Attest a key the application derived, named by algorithm | +| `Health` | Report whether the application is serving | + +Nothing here returns key material, and no caller chooses what gets signed. +`AttestAppKey` names a key by algorithm alone; the agent derives it and attests +the public key. A caller cannot build that report data itself, because it does +not know the public key until the agent derives it, which is why attesting an app +key needs its own method rather than the caller-supplied `report_data` that +`Attest` takes. + +`AttestAppKey` replaces the frozen `Worker.GetAttestationForAppKey`. That method +returns a `GetQuoteResponse`, which only Intel TDX can fill, so it fails on every +other platform and leaves an external caller there with no way to attest an app +key at all. `AttestAppKey` takes the same request and returns the v1 +`AttestResponse`, on every platform. Its `boottime_gpu_evidence` is always empty: +this method attests a key, not the machine. + +Both methods commit to the same public key for a given algorithm, because both +build their report data the same way. Only the envelope differs. + +`Health` is polled by the gateway to decide whether an instance belongs in its +application's load-balancing rotation. It answers from a cache the agent +refreshes on its own timer, so one call costs a lock and a clone however many +gateway nodes are polling. Only instances that opted in via +`RegisterCvmRequest.health_check` are ever polled; see +[Application health checks](./app-health-checks.md). + +### public_tcbinfo on the external surface + +`WorkerV1.Info` returns the same `InfoResponse` the internal surface returns, +minus what the application asked to keep private. Unless the app-compose sets +`public_tcbinfo`, the three document fields come back as empty strings: + +- `app_compose` +- `vm_config` +- `key_provider_info` + +Identity and the measurement hashes are always present. That is the same line the +unversioned `Worker.Info` drew, which blanked `tcb_info` and `vm_config` under +the same flag. + +The internal `DstackGuestV1.Info` applies no gating at all. The flag decides what +an outside party may learn, and the caller on the internal socket is the +application itself; an application cannot need protecting from its own +configuration. + +## Attestation + +`Attest` is v1's only CVM attestation entry point. It takes up to 64 bytes of +`report_data`, zero-padded on the right to 64; more than 64 bytes is an error +rather than a truncation. It returns the versioned dstack attestation format, +which covers every supported platform. + +### Extracting a quote from an attestation + +`AttestResponse.attestation` is a `VersionedAttestation`. The authoritative +implementation is `dstack/dstack-attest/src/v1.rs`, and `dstack-verifier` is the +reference consumer. + +The wire format is sniffed from the first byte. A leading `0x00` marks the legacy +SCALE-encoded V0 form; a MessagePack map prefix marks V1. V1 decodes as a +MessagePack map produced by `rmp_serde::to_vec_named`: + +```text +{ "version": u64, + "platform": { "kind": , "data": { ... } }, + "stack": { "kind": , "data": { ... } } } +``` + +`platform.kind` is one of `tdx`, `gcp-tdx`, `nitro-enclave`, `aws-nitro-tpm`, or +`sev-snp`. For `tdx` and `gcp-tdx`, `platform.data` carries `quote` (the raw TDX +quote bytes, exactly what the unversioned `GetQuote` returned) and `event_log`. +`gcp-tdx` additionally carries `tpm_quote`, which GCP's own verification binds and +which `GetQuote` had no field for. The other three platforms have no TDX quote, +which is why `GetQuote` could not answer on them at all. + +`stack.data.report_data` carries the 64 bytes the caller asked for. + +V2 runtime events in the event log always include the hex-encoded preimage of +their digest; a verifier should check that `sha384(hex_decode(preimage))` equals +the digest. + +### GPU evidence + +`AttestResponse.boottime_gpu_evidence` is populated only when the request sets +`include_boottime_gpu_evidence` and boot-time output exists. It is not bound to +`report_data`: nvattest ran at boot against its own nonce. To bind it, replay the +runtime event log and compare `sha256` of those exact UTF-8 bytes against the +`evidence_sha256` field of the measured `gpu-attestation` event. + +That evidence is a historical statement about the boot and does not prove the GPU +is still attached. Sampling the GPU at attestation time would not fix it, because +an NVIDIA report binds the device and a nonce but not the TD the device is +attached to, so a fresh report can be relayed from a genuine remote GPU. Only +TDISP/TEE-IO device binding closes that gap. + +`AttestGpu` answers the narrower question of whether the device reachable right +now is a genuine CC-enabled GPU that signs a caller-chosen 32-byte nonce. It +returns vendor-native evidence bundles rather than a local verdict, so a relying +party appraises them with its own verifier. + +## Info + +`Info` returns identity and configuration. It is not attestation, and nothing it +returns is evidence: the response arrives over a local socket with no quote behind +it. + +That is why the measurement registers and the event log are absent. v0's +`AppInfo.tcb_info` carried MRTD, RTMR0-3 and the event log inside a JSON string, +which invited relying parties to read measurements out of an unattested response. +Those values belong to `Attest`, whose attestation carries them quote-backed. + +`mr_aggregated`, `os_image_hash` and `compose_hash` remain, deliberately. They +identify *which* application and image this is, which is the question `Info` +answers. They are typed bytes rather than hex strings inside a JSON blob, and each +appears exactly once; v0 returned all three both as top-level fields and again, +hex-encoded, inside `tcb_info`. They are still unattested, and a relying party +still confirms them against an attestation. + +`app_cert` is gone. It was a self-issued demo certificate the agent minted for a +dashboard, and it proved nothing. + +`app_compose` carries the verbatim deployed document. `compose_hash` is `sha256` +over exactly those bytes, so do not parse and re-serialize before hashing: key +order, whitespace and unknown fields all change the digest, and that digest is +what gets whitelisted on chain. + +v1 applies no `public_tcbinfo` hiding. That flag decides what the *external* +listener may reveal, and v1 is on the internal socket only, so there is nobody to +hide from; the caller is the application itself. External exposure stays on the +unversioned `Worker.Info`. + +## Errors + +| Condition | Behaviour | +|---|---| +| Empty or unrecognised `algorithm` | Error naming the accepted values | +| Derived secp256k1 scalar out of range | Error; the caller picks another domain | +| `report_data` longer than 64 bytes | Error | +| `not_before` not earlier than `not_after` | Error | +| Unknown method, including `/v1` on an older agent | HTTP 400 | + +Every one of these is an HTTP 400 with the message in the body. v1 does not +default, coerce, or truncate a malformed request into a well-formed one. + +## Migration from the unversioned API + +**v1 keys are different keys.** Deriving under the same name on `/v1` that an +application used on `/` returns different key material. This is the point of the +new KDF, not a defect: the v0 derivation ignored the algorithm, so one secret +served two curves. There is no compatibility mode and no flag to get the old bytes +back from `/v1`. + +An application holding assets or identity under a v0 key must migrate them +deliberately. Derive the v1 key, move the asset with a transaction signed by the +v0 key, and only then cut over. An application with no persistent state can switch +by pointing at the new URL. + +Both unversioned surfaces stay available and closed. A v0.5.x client keeps +working against a 0.6 agent with no changes: `Sign`, `Verify` and `EmitEvent` +remain on the internal one (`EmitEvent` fails with a message naming its removal), +and `GetAttestationForAppKey` remains on the external one. Nothing forces a +migration. + +**SDK shape.** The SDKs ship two clients that mirror the two surfaces: a +`ClientV0` for the closed unversioned API, including its `Sign` and `Verify` RPCs, +and a `ClientV1` for this one. They are transport mirrors, not a compatibility +layer, and neither translates calls to the other. `ClientV1` has no `Sign` and no +`Verify` because v1 has neither; an application signs locally and a relying party +verifies locally, following the rules above. + +## Field mapping + +For readers porting from the unversioned API. + +| v0 | v1 | Note | +|---|---|---| +| `GetTlsKey` | `IssueCert` | Renamed; same behaviour | +| `GetKeyArgs.path` + `.purpose` | `GetKeyRequest.domain` | Merged; both KDF inputs now | +| `GetKeyArgs.algorithm` (defaulted) | `GetKeyRequest.algorithm` | Required; no `k256` alias | +| — | `GetKeyResponse.public_key` | Added | +| `GetQuote` | `Attest` | TDX-only channel subsumed | +| `AppInfo.tcb_info` | — | Measurements are typed fields; the rest belongs to `Attest` | +| `AppInfo.app_cert` | — | Dashboard artifact | +| `AppInfo.vm_config` | `InfoResponse.vm_config` | Unchanged content | +| `AppInfo.key_provider_info` | `InfoResponse.key_provider_info` | Unchanged content | +| (nested in `tcb_info`) | `InfoResponse.app_compose` | Promoted to top level | +| `Sign` | — | Removed; sign locally with the key from `GetKey` | +| `Verify` | — | Removed; verify locally per this document | +| `EmitEvent` | — | Removed; RTMR3 is system-owned | +| `Worker.GetAttestationForAppKey` | `WorkerV1.AttestAppKey` | Same request; answers on every platform | +| `Worker.Info` | `WorkerV1.Info` | Same `public_tcbinfo` gating | + +## Related documents + +- [Attestation on Intel TDX](./attestation-tdx.md) +- [Application health checks](./app-health-checks.md), for `WorkerV1.Health` +- [App Compose format](./normalized-app-compose.md), for the schema behind + `InfoResponse.app_compose` +- [On-chain governance](./onchain-governance.md), for the `DstackKms` contract that + publishes the KMS root public key From f1242022c753de19a5b37d1e254027a7e45500cd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 02:50:32 -0700 Subject: [PATCH 4/7] docs: changelog for the guest agent v1 API --- CHANGELOG.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8398f05ff..3538c926a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - sdk: `verify_signature` and `verify_signature_chain` in all four SDKs, replacing the deprecated guest-agent `Verify` RPC. `verify_signature_chain` is new capability rather than a port: it walks all three links of a `Sign` signature chain -- payload signature, the app root key attesting `"{purpose}:{hex(pubkey)}"`, and the KMS root attesting that app root for this `app_id` -- and requires the chain to anchor at a KMS root public key **the caller supplies**. That anchor has to come from somewhere independently trusted (the `DstackKms` contract's `kmsInfo().k256Pubkey`, or a pinned value); read it from the KMS being checked and an attacker who can answer that query can also mint a self-consistent chain. The four ports are pinned against one committed set of test vectors, `sdk/tests/vectors/signature_chain.json`, generated from the real KMS and guest-agent primitives -- this repo has shipped cross-language crypto drift twice already -- guest-agent: `AttestGpu` collects vendor-native GPU evidence on demand against a caller-supplied 32-byte nonce. It returns opaque, versioned evidence bundles identified by vendor and format for independent appraisal. The response format is extensible to additional GPU vendors. Exposed in the Rust, Python, Go, and JS SDKs -- guest-agent: `Attest` accepts `include_boottime_gpu_evidence` and returns the boot-time GPU attestation evidence in `AttestResponse.boottime_gpu_evidence`, so a verifier can fetch the quote and the GPU evidence in one round trip instead of also calling `GpuInfo`. Exposed in the Rust, Python, Go and JS SDKs +- guest-agent: a versioned API, `dstack.guest.v1`, covering both trust surfaces. `DstackGuestV1` is served at `/v1` on the internal socket (`IssueCert`, `GetKey`, `Attest`, `AttestGpu`, `Info`, `Version`); `WorkerV1` at `/prpc/v1` on the external listener (`Info`, `Version`, `AttestAppKey`, `Health`). They are two services rather than one mounted twice because the two listeners have different reachability: the internal socket answers only the app itself and hands out key material, the external one answers anyone who can route to the CVM and never does. Version selection is by URL path alone -- no header negotiation, no default-version redirect -- so a request URL is the whole record of which contract the caller asked for. Specified byte-for-byte in `docs/guest-api-v1.md`, which is the normative reference an implementation is written from +- guest-agent: v1 derives application keys under a real domain-separated KDF. v0 fed the path alone into HKDF and handed the same 32 bytes to both secp256k1 and ed25519, so the two curves shared one secret and the algorithm a caller asked for changed nothing about the key it got. v1 binds a versioned context tag, the algorithm and the caller's `domain` as length-prefixed fields, so the curves never collide and no two names encode alike -- a path is arbitrary caller-chosen bytes, so any delimiter it could also contain is a collision waiting to happen. **v1 keys are therefore not v0 keys for the same name**, deliberately and with no compatibility mode; an app holding assets under a v0 key migrates them with a transaction signed by the old key. The derivation is flat: `a/b` is not a child of `a`, and there is no BIP-32-style hierarchy. Committed test vectors pin the output bytes +- guest-agent: v1's signature-chain claim cannot be forged through the v0 surface. v0's first link signs `keccak256("{purpose}:{hex(pubkey)}")` over a caller-chosen `purpose`, which lets a malicious app steer the app root key into signing nearly any ASCII string ending in `:` plus hex. The v1 claim is length-prefixed and binds the raw public key bytes, so it always contains `00` bytes inside the region a v0 preimage requires to be hex-only. The exclusion is structural rather than probabilistic, and a regression test builds the strongest available forgery and asserts it fails +- guest-agent: v1 ships no `Sign` and no `Verify`. The agent is not an HSM: anything that can reach this socket can ask `GetKey` for the private key, so a server-side `Sign` grants no capability its caller lacks and buys an IPC round trip and another entry point to audit. Verifying needs neither key nor attestation, and the agent's answer arrives unattested. Apps sign locally with a standard library; relying parties verify locally against the normative rules in `docs/guest-api-v1.md`, which specify the KDF, the claim encoding, and every verification step down to the trust anchor. Both RPCs stay on the unversioned surface for 0.5.x clients +- guest-agent: v1 `GetTlsKey` is renamed `IssueCert`, because certificate issuance is the operation -- the agent builds a CSR and relays it to the KMS `SignCert` flow. The returned private key is incidental: freshly generated per call, fed by none of the request fields, and unrelated to the app identity. Only the integrated one-step mode ships; a caller-supplied-CSR mode would arrive as new fields +- guest-agent: v1 `Info` returns identity and configuration only. The `tcb_info` JSON blob is gone: its measurement values are typed top-level fields that each appear exactly once, and the documents it nested (`app_compose`, `vm_config`) are served directly instead of through two layers of JSON parsing. MRTD, RTMR0-3 and the event log are deliberately absent -- they are attestation data, and `Info` handing out unattested copies invited relying parties to trust values nothing vouched for. Ask `Attest` and verify. The demo `app_cert` is dropped as well +- guest-agent: `AttestGpu` collects vendor-native GPU evidence on demand against a caller-supplied 32-byte nonce. It returns opaque, versioned evidence bundles identified by vendor and format for independent appraisal, and the response format is extensible to additional GPU vendors. v1 only +- guest-agent: v1 `Attest` accepts `include_boottime_gpu_evidence` and returns the boot-time GPU attestation evidence in `AttestResponse.boottime_gpu_evidence`, so a verifier fetches the attestation and the GPU evidence in one round trip. `Attest` is also v1's sole CVM attestation entry point: the `VersionedAttestation` it returns already carries the TDX quote and event log, and unlike `GetQuote` it answers on every supported platform - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796) - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer - gateway: `Admin.SetInstanceReady` takes a CVM instance out of its app's load-balancing rotation without stopping it; instance-id routing stays open so the instance can still be investigated, and the setting survives re-registration - gateway: operator-set per-instance overrides now live under their own KV keys — `admin//ready` and `admin//port_policy` — instead of inside the instance record, so a CVM re-registration can no longer drop them and setting one cannot discard a peer's unsynced change to the other. An override left in an instance record by an earlier build is moved across on load -- gateway: opt-in application-level health polling. An app sets `requirements.health_check` in its app-compose; the gateway then asks that CVM's guest agent (new `Worker.Health` RPC) whether the app is serving, and keeps instances that say no -- or that have not answered since registering -- out of app-id load balancing. Apps that do not opt in are never polled. Instance-id routing is never gated, and an app whose every instance reports unhealthy is routed to anyway rather than blackholed. Verdicts are per-node and not persisted: a gateway restart puts the whole app back at `unknown` at once, which is exactly the case the fail-open covers. Documented in `docs/app-health-checks.md` +- gateway: opt-in application-level health polling. An app sets `requirements.health_check` in its app-compose; the gateway then asks that CVM's guest agent (new `WorkerV1.Health` RPC, polled at `/prpc/v1`) whether the app is serving, and keeps instances that say no -- or that have not answered since registering -- out of app-id load balancing. Apps that do not opt in are never polled. Instance-id routing is never gated, and an app whose every instance reports unhealthy is routed to anyway rather than blackholed. Verdicts are per-node and not persisted: a gateway restart puts the whole app back at `unknown` at once, which is exactly the case the fail-open covers. Documented in `docs/app-health-checks.md` - app-compose: `requirements.health_status_file` names a file the app writes its own verdict into -- two lines, `healthy`/`unhealthy` and the unix timestamp it was written at, treated as unhealthy once older than 60s. It must be a regular file (a FIFO would park a thread of the agent's blocking pool on every refresh); symlinks are followed, and its contents are never quoted back into a report. Without it the agent judges the app's own Compose project: every container that declares a `healthcheck` must be running and healthy, and a project where *no* container declares one reports unhealthy rather than passing silently - guest-agent: container health also covers the `nerdctl-compose` runner, read through `nerdctl inspect` (its output is Docker-compatible). Requires nerdctl >= 2.3.1 for Compose `healthcheck:` to be honoured; the mkosi backend is pinned to 2.3.5 - http-client: a caller can bound the response body (`http_request_bounded`, `PrpcClient::with_max_response_bytes`). Nothing is bounded by default — `dstack vmm logs --lines 100000` is a legitimate multi-megabyte fetch — but every client that talks to a guest agent opts in, in the gateway and in the VMM, because a CVM is untrusted and one of them polls on a timer against the whole fleet @@ -38,8 +44,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - vmm: optionally randomize the KMS and gateway URL orders written to each CVM's system configuration so new CVMs distribute their initial requests across service nodes; both are enabled by default in `vmm.toml` - http-client: HTTP clients are built once and shared instead of per request, which is what every `http_request*` call did until now -- each one paid for a connection pool, a DNS resolver and a TLS configuration it then threw away. Callers choose whether requests may reuse a connection (`RequestOptions::connection_reuse`, `PrpcClient::with_connection_reuse`); the default is to reuse. The gateway's health poller opts out: opening the connection is half of what a probe asks, since an agent that has run out of file descriptors keeps serving connections it already has while refusing every new one -- and every connection the gateway proxies to an app is a new one - os/yocto: nerdctl 2.2.1 → 2.3.5, so `nerdctl compose` honours the Compose `healthcheck:` field (only translated into `--health-*` flags from 2.3.1 on). Requires openembedded-core to move to `wrynose` head for go 1.26.5, which also brings gcc 15.2 → 15.3 — every guest image measurement changes, so the new image hashes need whitelisting in KMS +- guest-agent: both unversioned surfaces are closed at exactly v0.5.11, and every new capability goes to `dstack.guest.v1` instead. Everything added to them after v0.5.11 never shipped in a release, so it is removed rather than frozen in: on the internal `DstackGuest`, `GpuInfo`, `AttestGpu` and `Attest`'s `include_boottime_gpu_evidence`/`boottime_gpu_evidence`; on the external `Worker`, `AttestAppKey` and `Health`. `AttestGpu`, the boot-time evidence, `AttestAppKey` and `Health` are v1 features now; `GpuInfo` is gone entirely, because `Attest(include_boottime_gpu_evidence)` returns the same bytes. No released client is affected -- both services are now byte-identical to v0.5.11 apart from doc comments and `reserved` statements holding the interim field numbers, so an unreleased `next` build in a dev environment cannot have one of them silently absorbed by a future field. "Frozen except for additions" is how all of them arrived in the first place - guest-agent: `GetQuote` is restricted to Intel TDX. It used to answer on every platform, returning an empty `quote` plus a `GetQuoteResponse.attestation` field carrying the versioned attestation — a shape only that one RPC produced, and one `Attest` already covers. Platforms without TDX now get an error telling them to call `Attest`, and the `attestation` field is gone from the RPC and from the Rust, Python, Go and JS SDKs. GCP Confidential VMs still get an answer — the gate is whether the platform has a TDX quote — but only the TDX half of one: `GetQuoteResponse` has no field for the vTPM quote GCP's verification also binds, so relying parties there want `Attest`, and the docs say so. `Tappd.TdxQuote`/`RawQuote` share the same backend path, so they fail closed there too instead of returning an empty quote -- guest-agent: `Worker.GetAttestationForAppKey` is replaced by `Worker.AttestAppKey`. The old method returned a `GetQuoteResponse`, so restricting `GetQuote` to Intel TDX left it unable to answer anywhere else — and the external listener with no way to attest an app key at all, since `Attest` is on the internal socket and an external caller could not use it anyway, not knowing the app key's public key until the agent derives it. `AttestAppKey` takes the same request and returns an `AttestResponse`, on every platform. This is a breaking change to an RPC present since v0.5.7; it ships no SDK method and has no known callers +- guest-agent: `Worker.GetAttestationForAppKey` is **retained** on the frozen external surface, and `WorkerV1.AttestAppKey` supersedes it. The old method returns a `GetQuoteResponse`, so restricting `GetQuote` to Intel TDX leaves it unable to answer anywhere else, which would have left an external caller on any other platform with no way to attest an app key at all -- `Attest` is on the internal socket, and an external caller could not use it anyway, not knowing the app key's public key until the agent derives it. Rather than change an RPC present since v0.5.7, it keeps its v0.5.11 wire shape and stays TDX-only, and `AttestAppKey` on the v1 external surface takes the same request and returns an `AttestResponse` on every platform. Both build their report data through the same code, so they commit to the same public key for a given algorithm; only the envelope differs ### Removed From f4fea397e2ff7dde0539f9cceb4b0c3ebfb170cb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 03:01:20 -0700 Subject: [PATCH 5/7] feat(guest-agent): name the frozen surfaces v0 and mount them at /v0 --- dstack/guest-agent/rpc/proto/agent_rpc.proto | 18 ++- dstack/guest-agent/src/server.rs | 142 ++++++++++++++++--- 2 files changed, 134 insertions(+), 26 deletions(-) diff --git a/dstack/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto index eb1a62bc9..cc6d5bfaf 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc.proto @@ -36,10 +36,13 @@ service Tappd { // The service for the dstack guest agent. // -// This unversioned surface is CLOSED. It is exactly the v0.5.11 surface and -// stays that way: no additions, no renumbering, no removals, and no semantic -// changes to existing methods. It exists so a v0.5.x client keeps working -// against a 0.6 agent unchanged. +// This surface is CLOSED. It is exactly the v0.5.11 surface and stays that way: +// no additions, no renumbering, no removals, and no semantic changes to +// existing methods. It exists so a v0.5.x client keeps working against a 0.6 +// agent unchanged. +// +// It is the `v0` surface, served at `/v0`. Its historical unversioned path `/` +// stays mounted as an alias onto this same handler for pre-0.6 clients. // // Every new capability goes to `dstack.guest.v1` (agent_rpc_v1.proto), served // at `/v1` on the same socket. Do not add a method here, and do not add a @@ -304,9 +307,10 @@ message WorkerVersion { // The external guest agent service. // -// CLOSED, like `DstackGuest`: exactly the v0.5.11 surface, served at `/prpc` -// on the external listener. New capability goes to `dstack.guest.v1`'s -// `WorkerV1`, served at `/prpc/v1` on the same listener. +// CLOSED, like `DstackGuest`: exactly the v0.5.11 surface, served at +// `/prpc/v0` on the external listener, with `/prpc` kept as an alias onto the +// same handler for pre-0.6 clients. New capability goes to `dstack.guest.v1`'s +// `WorkerV1`, served at `/prpc/v1`. service Worker { // Get app info rpc Info(google.protobuf.Empty) returns (AppInfo) {} diff --git a/dstack/guest-agent/src/server.rs b/dstack/guest-agent/src/server.rs index 8551e85bc..20ce2b8ba 100644 --- a/dstack/guest-agent/src/server.rs +++ b/dstack/guest-agent/src/server.rs @@ -16,6 +16,7 @@ use rocket::{ fairing::AdHoc, figment::Figment, listener::{unix::UnixListener, Bind, DefaultListener, Endpoint}, + Build, Rocket, }; use rocket_vsock_listener::VsockListener; use sd_notify::{notify as sd_notify, NotifyState}; @@ -81,20 +82,57 @@ async fn run_internal_v0( Ok(()) } +/// Mount everything the internal socket serves. +/// +/// `/v0` is the name of the frozen v0.5.11 surface and `/v1` is the current +/// one. `/` is the same frozen surface under its historical path, kept as a +/// compatibility alias so a pre-0.6 client keeps working unchanged -- it is the +/// identical handler, not a copy, so the two paths cannot drift. +/// +/// Selection is by URL path alone: no header negotiation, no default-version +/// redirect, so a caller's URL is the whole record of which contract it asked +/// for. +/// +/// Factored out of `run_internal` so a test can exercise the real mount table +/// rather than a restatement of it. +fn mount_internal(rocket: Rocket) -> Rocket { + rocket + .mount("/", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) + .mount("/v0", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) + .mount("/v1", ra_rpc::prpc_routes!(AppState, V1RpcHandler)) +} + +/// Mount the pRPC services the external listener serves. +/// +/// Same scheme as the internal socket, one level down: `/prpc/v0` is the frozen +/// v0.5.11 `Worker`, `/prpc/v1` is `WorkerV1`, and `/prpc` is the frozen surface +/// under its historical path. +/// +/// The `trim` on the frozen mounts strips the service name a pre-0.6 client +/// prefixes, so `/prpc/Worker.Info` and `/prpc/Info` both land on `Info`. +fn mount_external(rocket: Rocket) -> Rocket { + rocket + .mount( + "/prpc", + ra_rpc::prpc_routes!(AppState, ExternalRpcHandler, trim: "Worker."), + ) + .mount( + "/prpc/v0", + ra_rpc::prpc_routes!(AppState, ExternalRpcHandler, trim: "Worker."), + ) + .mount( + "/prpc/v1", + ra_rpc::prpc_routes!(AppState, ExternalV1RpcHandler), + ) +} + async fn run_internal( state: AppState, figment: Figment, activated_socket: Option, sock_ready_tx: oneshot::Sender<()>, ) -> Result<()> { - // Two surfaces, one socket, selected by URL path alone. `/` is the frozen - // unversioned API v0.5.x clients speak; `/v1` is `dstack.guest.v1`. There - // is no header negotiation and no default-version redirect, so a caller's - // URL is the whole record of which contract it asked for. - let rocket = rocket::custom(figment) - .mount("/", ra_rpc::prpc_routes!(AppState, InternalRpcHandler)) - .mount("/v1", ra_rpc::prpc_routes!(AppState, V1RpcHandler)) - .manage(state); + let rocket = mount_internal(rocket::custom(figment)).manage(state); let ignite = rocket .ignite() .await @@ -139,18 +177,8 @@ async fn run_internal( } async fn run_external(state: AppState, figment: Figment) -> Result<()> { - let rocket = rocket::custom(figment) + let rocket = mount_external(rocket::custom(figment)) .mount("/", http_routes::external_routes(state.config())) - // Same two-surface split as the internal socket: `/prpc` is the - // unversioned Worker, closed at v0.5.11, and `/prpc/v1` is `WorkerV1`. - .mount( - "/prpc", - ra_rpc::prpc_routes!(AppState, ExternalRpcHandler, trim: "Worker."), - ) - .mount( - "/prpc/v1", - ra_rpc::prpc_routes!(AppState, ExternalV1RpcHandler), - ) .attach(AdHoc::on_response("Add app version header", |_req, res| { Box::pin(async move { res.set_raw_header("X-App-Version", app_version()); @@ -269,3 +297,79 @@ pub async fn run(state: AppState, figment: Figment, watchdog: bool) -> Result<() ); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_service::tests::setup_test_state; + use rocket::local::asynchronous::Client; + + /// Fetch `path` from a rocket carrying the real mount table. + async fn get(mount: fn(Rocket) -> Rocket, path: &str) -> (u16, String) { + let (state, _guard) = setup_test_state().await; + let client = Client::tracked(mount(rocket::build()).manage(state)) + .await + .expect("rocket failed to ignite"); + let response = client.get(path).dispatch().await; + let status = response.status().code; + (status, response.into_string().await.unwrap_or_default()) + } + + /// The frozen surface answers identically on `/v0` and on the unversioned + /// path it has always had. + /// + /// The alias is what lets a pre-0.6 client keep working, so it has to stay + /// the same handler rather than a second one that happens to agree today. + #[tokio::test] + async fn the_internal_v0_mount_is_an_alias_for_the_unversioned_path() { + let (unversioned_status, unversioned) = get(mount_internal, "/Version").await; + let (v0_status, v0) = get(mount_internal, "/v0/Version").await; + + assert_eq!(unversioned_status, 200, "{unversioned}"); + assert_eq!(v0_status, 200, "{v0}"); + assert_eq!(unversioned, v0); + assert!(v0.contains("version"), "{v0}"); + } + + /// Same on the external listener, one level down. + #[tokio::test] + async fn the_external_v0_mount_is_an_alias_for_the_unversioned_path() { + let (unversioned_status, unversioned) = get(mount_external, "/prpc/Version").await; + let (v0_status, v0) = get(mount_external, "/prpc/v0/Version").await; + + assert_eq!(unversioned_status, 200, "{unversioned}"); + assert_eq!(v0_status, 200, "{v0}"); + assert_eq!(unversioned, v0); + } + + /// A pre-0.6 client prefixes the service name. That has to keep working on + /// the alias and on `/v0`. + #[tokio::test] + async fn the_external_mounts_accept_the_service_name_prefix() { + for path in ["/prpc/Worker.Version", "/prpc/v0/Worker.Version"] { + let (status, body) = get(mount_external, path).await; + assert_eq!(status, 200, "{path}: {body}"); + } + } + + /// The version in the path selects the surface, and nothing else does. + /// `/v1` must not answer a method only the frozen surface has, and `/v0` + /// must not answer a v1-only one. + #[tokio::test] + async fn each_mount_serves_only_its_own_surface() { + // `Verify` is frozen-only; `IssueCert` is v1-only. + let (status, _) = get(mount_internal, "/v1/Verify").await; + assert_ne!(status, 200, "/v1 must not serve the frozen Verify"); + + for path in ["/IssueCert", "/v0/IssueCert"] { + let (status, _) = get(mount_internal, path).await; + assert_ne!(status, 200, "{path} must not serve the v1 IssueCert"); + } + + // `Health` is v1-only on the external listener. + for path in ["/prpc/Health", "/prpc/v0/Health"] { + let (status, _) = get(mount_external, path).await; + assert_ne!(status, 200, "{path} must not serve the v1 Health"); + } + } +} From a09e49e6e88a1dd3020246ebec9e0d623eb5de18 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 03:01:25 -0700 Subject: [PATCH 6/7] feat(guest-agent): derive v1 keys under a v1-specific HKDF salt --- .../guest-agent/rpc/proto/agent_rpc_v1.proto | 15 ++-- dstack/guest-agent/src/rpc_service_v1/keys.rs | 83 ++++++++++++++----- dstack/ra-tls/src/kdf.rs | 47 ++++++++++- 3 files changed, 116 insertions(+), 29 deletions(-) diff --git a/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto b/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto index 86505a786..88b7bb934 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc_v1.proto @@ -13,9 +13,10 @@ package dstack.guest.v1; // `DstackGuestV1` internal socket (`/var/run/dstack.sock`), at `/v1` // `WorkerV1` external listener, at `/prpc/v1` // -// Each unversioned surface stays mounted where it was and is closed at exactly -// v0.5.11. The version a caller gets is decided by the URL path alone, never by -// a header. +// The frozen v0.5.11 services are named `v0` and served at `/v0` and +// `/prpc/v0`; their historical unversioned paths (`/` and `/prpc`) stay mounted +// as aliases onto the same handlers, so a pre-0.6 client keeps working. The +// version a caller gets is decided by the URL path alone, never by a header. // // The two are not the same surface with different mounts. The internal socket // is reachable only by the application itself, so `DstackGuestV1` hands out key @@ -31,10 +32,10 @@ package dstack.guest.v1; // - a key is named by `(domain, algorithm)` and nothing else. There is no // `purpose` field anywhere in v1. Encode roles in the domain string. // -// v1 keys are NOT the v0 keys. The v1 KDF binds the algorithm and a versioned -// context tag alongside the domain, so the same name yields different key -// material here than on the unversioned surface, and secp256k1 and ed25519 no -// longer share one 32-byte secret. See `docs/guest-api-v1.md` for the +// v1 keys are NOT the v0 keys. The v1 KDF derives under its own HKDF salt and +// binds the algorithm and a versioned context tag alongside the domain, so the +// same name yields different key material here than on the frozen surface, and +// secp256k1 and ed25519 no longer share one 32-byte secret. See `docs/guest-api-v1.md` for the // byte-level construction. // // v1 serves only what genuinely needs the TEE: deriving keys from the app root diff --git a/dstack/guest-agent/src/rpc_service_v1/keys.rs b/dstack/guest-agent/src/rpc_service_v1/keys.rs index f595176bd..cddef0bed 100644 --- a/dstack/guest-agent/src/rpc_service_v1/keys.rs +++ b/dstack/guest-agent/src/rpc_service_v1/keys.rs @@ -13,9 +13,19 @@ use anyhow::{anyhow, bail, Context, Result}; use ed25519_dalek::SigningKey as Ed25519SigningKey; use k256::ecdsa::SigningKey; -use ra_tls::kdf::derive_key; +use ra_tls::kdf::derive_key_with_salt; use sha3::{Digest, Keccak256}; +/// The HKDF salt for every v1 derivation. +/// +/// Distinct from the legacy `RATLS` salt, which gives v1 its own derivation +/// tree rather than a differently-labelled branch of the old one. Under a shared +/// salt the two surfaces are separated only by their HKDF `info`, and the +/// legacy `info` is the caller's `path` verbatim -- so a caller that passed the +/// v1 `info` byte string as a v0 path would reproduce a v1 key. Different salts +/// close that by construction, whatever either side puts in `info`. +pub(crate) const KDF_SALT: &[u8] = b"dstack-guest-v1"; + /// Context tag bound into every v1 key derivation. pub(crate) const KEY_CONTEXT_TAG: &[u8] = b"dstack-guest-v1-key"; @@ -118,16 +128,17 @@ pub(crate) struct AppKey { impl AppKey { /// Derive the key for `(domain, algorithm)` from the app root key. /// - /// HKDF-SHA256 over the app root secp256k1 key, with the `info` from - /// [`key_derivation_info`]. Same primitive and same salt as v0; what - /// changed is that the algorithm and a version tag are now inputs, so the - /// two curves no longer share one secret and a v1 domain is not a v0 path. + /// HKDF-SHA256 over the app root secp256k1 key, under [`KDF_SALT`], with + /// the `info` from [`key_derivation_info`]. Same primitive as v0; what + /// changed is the salt, and that the algorithm and a version tag are now + /// inputs -- so the two curves no longer share one secret and a v1 domain + /// is not a v0 path. /// /// Flat, not hierarchical: `a/b` is an opaque domain string like any /// other, unrelated to `a`, and no key here derives another. pub(crate) fn derive(app_root_key: &[u8], domain: &str, algorithm: Algorithm) -> Result { let info = key_derivation_info(domain, algorithm)?; - let derived = derive_key(app_root_key, &[&info], 32) + let derived = derive_key_with_salt(KDF_SALT, app_root_key, &[&info], 32) .map_err(|_| anyhow!("failed to derive the application key"))?; let secret: [u8; 32] = derived .as_slice() @@ -189,6 +200,7 @@ impl AppKey { #[cfg(test)] mod tests { use super::*; + use ra_tls::kdf::derive_key; /// The app root key the committed vectors were generated from. Same value /// the handler tests use, so a vector can be reproduced end to end. @@ -256,40 +268,40 @@ mod tests { ( "", Algorithm::Secp256k1, - "463e877bc7322c1c09e567844b3101e88f353bfb33177c41cb13832cb67eef1c", - "03c45e036d19662802e628d9a712c07a9d9d64bee28e1754a72d75010860a789c2", + "59f60584ce6fd2a3a31997256db9d77322463fc8a6b1520110401bcb1ee92387", + "0377c7fb050db181d392266a3cee9adb2901c6d665f11bac68be5457f577ba4908", ), ( "", Algorithm::Ed25519, - "a41f6458de9d11a43f79640b6ab2c62d02aceda7b16c5f159dc7ad69621c7eb0", - "7a5740fa9ab4791a232cc1fc7e73d5ff47ad41be2589a71b05f128dee4223f08", + "b023493030669cf22e9cafa6a464d4cf3ae4edfe5474ec796710f21ea011946d", + "a3dc149fd5b765eab2eb7d3174fa939e39386898f10b15b7b146f6f1358ecf2a", ), ( "wallet", Algorithm::Secp256k1, - "c2b47271c2956c020eb471f3d6ec9a08bac4ce72d158078592c3bfd9db67808c", - "0375b11b6fabbe6e18b9bac26b082070dea76487ce512323870bc784a28ec5404b", + "2580611f0f936abe59399a8ac4ed9964d0259bd34c88ea012ca42b32acbf9386", + "0369cecd3c8da88730f7d45875824c3e75f63a2d3da4be42f45671954daa2abb28", ), ( "wallet", Algorithm::Ed25519, - "fce7a47f848fc9f7a799a34e061f4d07d7e8ef3adda98d8fc36a5de3f5146cdf", - "5e8076b492634770e9b12dc8b136c9f5e3e8a86adc657040ba7a320296dcf6b0", + "d76a703b08ebb074b809b9d6acf3d7c6663131273807717ce9d23bbadc2c644e", + "dade622d0fa1641e79b16e0b04e296be671f85f0aa6387b7d37e9d89f87494f5", ), ( "a/b/c", Algorithm::Secp256k1, - "448f92c86abd72c1c35b7ff75b5c9971114694e7825518e5ed0d2101711527cd", - "03011845be1d30004c148ae49ef3a82585094f856a7c9bc3ed06edf959537a4eac", + "7f0973449298085d2d36a3b4c4d3243c100ba1981ffa885fe9e9dee883e69538", + "02e9b1a61b6d70aa9b241753828c316bf90e33e77b2e113f9ba75a8b6dc3cde5c1", ), // A domain carrying NUL and `:`, the two characters a // delimiter-joined encoding would choke on. ( "k\u{0}:ey", Algorithm::Ed25519, - "3cfcf09543e94d23c87aec1e5774ca3803feb3dd8ee298507650409b20ebfdde", - "9ba2d94591b97db4f089fc187cfcb09a9cd55bdd553e066d80cc8cb2977bd841", + "42da8bf0b479ed125c370e3b91f982735bf08ff592abbd586985affa43ee96a1", + "c833107822b003ff5675b33b90b151d4315c3ab9162b17d876e8dffde41abf9b", ), ]; for (domain, algorithm, expected_secret, expected_public) in vectors { @@ -316,8 +328,8 @@ mod tests { let key = derive("wallet", Algorithm::Secp256k1); assert_eq!( hex::encode(key.claim_signature(&TEST_APP_ROOT_KEY).unwrap()), - "96726db35263cb2a7067fc5ebb0d06621f7cab2d0f0aa15a83858dbf9ff7f12c\ - 6e1c5640223fcca1a03ba5ccf09d353609db6481f9563add16e16a8ad8544c29\ + "af26d2f258d34580e7288bd83fc97bddc83769476d77823c4f76a3ad77a75149\ + 1a39ffc4ef3aa66cb0d008b8f6f199e6d57c1da9a92ba4cf10f23bf752b8cad0\ 00" ); } @@ -345,6 +357,37 @@ mod tests { } } + /// The one v0 path that could reach a v1 key under a shared salt: the + /// legacy `info` is the caller's `path` verbatim, so passing the v1 `info` + /// byte string as a v0 path made the two derivations identical. + /// + /// The v1 salt closes it by construction. This is the test that would have + /// failed before the salt changed, so it is the one that keeps it changed. + #[test] + fn a_v0_path_cannot_reproduce_a_v1_key() { + for (domain, algorithm) in [ + ("", Algorithm::Secp256k1), + ("wallet", Algorithm::Secp256k1), + ("wallet", Algorithm::Ed25519), + ] { + let info = key_derivation_info(domain, algorithm).unwrap(); + // The best a v0 caller can do: hand the whole v1 info to `path`. + let v0 = derive_key(&TEST_APP_ROOT_KEY, &[&info], 32).unwrap(); + assert_ne!( + derive(domain, algorithm).secret(), + v0, + "a v0 path reproduced the v1 key for ({domain:?}, {})", + algorithm.name() + ); + } + } + + #[test] + fn the_v1_salt_is_not_the_legacy_salt() { + assert_eq!(KDF_SALT, b"dstack-guest-v1"); + assert_ne!(KDF_SALT, ra_tls::kdf::LEGACY_SALT); + } + #[test] fn encodes_the_claim_with_the_raw_public_key() { let key = derive("wallet", Algorithm::Secp256k1); diff --git a/dstack/ra-tls/src/kdf.rs b/dstack/ra-tls/src/kdf.rs index 2b1aa1145..15303bdd5 100644 --- a/dstack/ra-tls/src/kdf.rs +++ b/dstack/ra-tls/src/kdf.rs @@ -19,13 +19,34 @@ impl KeyType for AnySizeKey { } } -/// Derives a key using HKDF-SHA256. +/// The salt every pre-0.6 derivation uses. +/// +/// Load-bearing: it is baked into every key deployed before the versioned guest +/// API. Never change it. +pub const LEGACY_SALT: &[u8] = b"RATLS"; + +/// Derives a key using HKDF-SHA256 under the legacy [`LEGACY_SALT`]. pub fn derive_key( input_key_material: &[u8], context_data: &[&[u8]], key_size: usize, ) -> Result, Unspecified> { - let salt = Salt::new(HKDF_SHA256, b"RATLS"); + derive_key_with_salt(LEGACY_SALT, input_key_material, context_data, key_size) +} + +/// Derives a key using HKDF-SHA256 under an explicit salt. +/// +/// A distinct salt gives a genuinely separate derivation tree: two callers using +/// different salts cannot land on the same key however their `context_data` +/// happens to be built, which a shared salt cannot promise when one caller's +/// context is attacker-chosen. +pub fn derive_key_with_salt( + salt: &[u8], + input_key_material: &[u8], + context_data: &[&[u8]], + key_size: usize, +) -> Result, Unspecified> { + let salt = Salt::new(HKDF_SHA256, salt); let pseudo_rand_key: Prk = salt.extract(input_key_material); let output_key_material: Okm = pseudo_rand_key.expand(context_data, AnySizeKey(key_size))?; @@ -102,6 +123,28 @@ mod tests { assert!(key.iter().any(|&x| x != 0)); } + /// The refactor that introduced an explicit salt must not have moved the + /// legacy derivation, which every pre-0.6 key depends on. + #[test] + fn derive_key_still_uses_the_legacy_salt() { + let context = [b"context one".as_ref(), b"context two".as_ref()]; + assert_eq!( + derive_key(b"input key material", &context, 32).unwrap(), + derive_key_with_salt(b"RATLS", b"input key material", &context, 32).unwrap() + ); + } + + /// Two salts, one everything else: the outputs must be unrelated. This is + /// what makes a salt change a real domain separation rather than a rename. + #[test] + fn a_different_salt_gives_a_different_key() { + let context = [b"context one".as_ref()]; + assert_ne!( + derive_key_with_salt(b"RATLS", b"ikm", &context, 32).unwrap(), + derive_key_with_salt(b"dstack-guest-v1", b"ikm", &context, 32).unwrap() + ); + } + #[test] fn test_derive_key256() { let key = derive_key(b"input key material", &[b"context one"], 256).unwrap(); From 5982b85affd72939a65b66d9036f1930ef28acdd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 03:01:25 -0700 Subject: [PATCH 7/7] docs: specify the v0/v1 path scheme and the v1 KDF salt --- CHANGELOG.md | 4 +-- docs/guest-api-v1.md | 73 ++++++++++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3538c926a..e91453ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - sdk: `verify_signature` and `verify_signature_chain` in all four SDKs, replacing the deprecated guest-agent `Verify` RPC. `verify_signature_chain` is new capability rather than a port: it walks all three links of a `Sign` signature chain -- payload signature, the app root key attesting `"{purpose}:{hex(pubkey)}"`, and the KMS root attesting that app root for this `app_id` -- and requires the chain to anchor at a KMS root public key **the caller supplies**. That anchor has to come from somewhere independently trusted (the `DstackKms` contract's `kmsInfo().k256Pubkey`, or a pinned value); read it from the KMS being checked and an attacker who can answer that query can also mint a self-consistent chain. The four ports are pinned against one committed set of test vectors, `sdk/tests/vectors/signature_chain.json`, generated from the real KMS and guest-agent primitives -- this repo has shipped cross-language crypto drift twice already -- guest-agent: a versioned API, `dstack.guest.v1`, covering both trust surfaces. `DstackGuestV1` is served at `/v1` on the internal socket (`IssueCert`, `GetKey`, `Attest`, `AttestGpu`, `Info`, `Version`); `WorkerV1` at `/prpc/v1` on the external listener (`Info`, `Version`, `AttestAppKey`, `Health`). They are two services rather than one mounted twice because the two listeners have different reachability: the internal socket answers only the app itself and hands out key material, the external one answers anyone who can route to the CVM and never does. Version selection is by URL path alone -- no header negotiation, no default-version redirect -- so a request URL is the whole record of which contract the caller asked for. Specified byte-for-byte in `docs/guest-api-v1.md`, which is the normative reference an implementation is written from -- guest-agent: v1 derives application keys under a real domain-separated KDF. v0 fed the path alone into HKDF and handed the same 32 bytes to both secp256k1 and ed25519, so the two curves shared one secret and the algorithm a caller asked for changed nothing about the key it got. v1 binds a versioned context tag, the algorithm and the caller's `domain` as length-prefixed fields, so the curves never collide and no two names encode alike -- a path is arbitrary caller-chosen bytes, so any delimiter it could also contain is a collision waiting to happen. **v1 keys are therefore not v0 keys for the same name**, deliberately and with no compatibility mode; an app holding assets under a v0 key migrates them with a transaction signed by the old key. The derivation is flat: `a/b` is not a child of `a`, and there is no BIP-32-style hierarchy. Committed test vectors pin the output bytes +- guest-agent: a versioned API, `dstack.guest.v1`, covering both trust surfaces. `DstackGuestV1` is served at `/v1` on the internal socket (`IssueCert`, `GetKey`, `Attest`, `AttestGpu`, `Info`, `Version`); `WorkerV1` at `/prpc/v1` on the external listener (`Info`, `Version`, `AttestAppKey`, `Health`). They are two services rather than one mounted twice because the two listeners have different reachability: the internal socket answers only the app itself and hands out key material, the external one answers anyone who can route to the CVM and never does. The scheme is uniform across both listeners: `/v0` and `/v1` on the internal socket, `/prpc/v0` and `/prpc/v1` on the external one, where `v0` is the frozen v0.5.11 surface. The historical unversioned paths (`/` and `/prpc`) stay mounted as aliases onto the same frozen handlers, so a pre-0.6 client keeps working unchanged and cannot drift from `/v0` -- they are additional mounts, not a parallel implementation. Version selection is by URL path alone -- no header negotiation, no default-version redirect -- so a request URL is the whole record of which contract the caller asked for. `Tappd` predates this scheme and is untouched. Specified byte-for-byte in `docs/guest-api-v1.md`, which is the normative reference an implementation is written from +- guest-agent: v1 derives application keys under a real domain-separated KDF. v0 fed the path alone into HKDF and handed the same 32 bytes to both secp256k1 and ed25519, so the two curves shared one secret and the algorithm a caller asked for changed nothing about the key it got. v1 derives under its own HKDF salt (`dstack-guest-v1`, against the legacy `RATLS`) and binds a versioned context tag, the algorithm and the caller's `domain` as length-prefixed fields, so the curves never collide and no two names encode alike -- a path is arbitrary caller-chosen bytes, so any delimiter it could also contain is a collision waiting to happen. **v1 keys are therefore not v0 keys for the same name**, deliberately and with no compatibility mode; an app holding assets under a v0 key migrates them with a transaction signed by the old key. The separate salt matters because the legacy HKDF `info` is the caller's `path` verbatim: under a shared salt, a caller passing the v1 `info` byte string as a v0 `path` reproduced a v1 key exactly. That was never a privilege boundary -- same app, same root key, either surface reachable -- but a KDF whose separation depends on nobody choosing an awkward input is one refactor from separating nothing, and it costs nothing to close on a surface with no deployed keys. The derivation is flat: `a/b` is not a child of `a`, and there is no BIP-32-style hierarchy. Committed test vectors pin the output bytes - guest-agent: v1's signature-chain claim cannot be forged through the v0 surface. v0's first link signs `keccak256("{purpose}:{hex(pubkey)}")` over a caller-chosen `purpose`, which lets a malicious app steer the app root key into signing nearly any ASCII string ending in `:` plus hex. The v1 claim is length-prefixed and binds the raw public key bytes, so it always contains `00` bytes inside the region a v0 preimage requires to be hex-only. The exclusion is structural rather than probabilistic, and a regression test builds the strongest available forgery and asserts it fails - guest-agent: v1 ships no `Sign` and no `Verify`. The agent is not an HSM: anything that can reach this socket can ask `GetKey` for the private key, so a server-side `Sign` grants no capability its caller lacks and buys an IPC round trip and another entry point to audit. Verifying needs neither key nor attestation, and the agent's answer arrives unattested. Apps sign locally with a standard library; relying parties verify locally against the normative rules in `docs/guest-api-v1.md`, which specify the KDF, the claim encoding, and every verification step down to the trust anchor. Both RPCs stay on the unversioned surface for 0.5.x clients - guest-agent: v1 `GetTlsKey` is renamed `IssueCert`, because certificate issuance is the operation -- the agent builds a CSR and relays it to the KMS `SignCert` flow. The returned private key is incidental: freshly generated per call, fed by none of the request fields, and unrelated to the app identity. Only the integrated one-step mode ships; a caller-supplied-CSR mode would arrive as new fields diff --git a/docs/guest-api-v1.md b/docs/guest-api-v1.md index 2956087b9..1e7cdb7a2 100644 --- a/docs/guest-api-v1.md +++ b/docs/guest-api-v1.md @@ -51,13 +51,25 @@ Every surface the agent serves is now a closed v0 fossil plus a v1. The unversioned surfaces are exactly v0.5.11 and never change again; new capability arrives only in `dstack.guest.v1`. -| Listener | Service | Mount | Example path | -|---|---|---|---| -| Internal socket | `DstackGuestV1` | `/v1` | `/v1/GetKey` | -| Internal socket | `DstackGuest`, closed | `/` | `/GetKey` | -| Internal socket | `Tappd`, closed | `/prpc/` | `/prpc/Tappd.TdxQuote` | -| External | `WorkerV1` | `/prpc/v1` | `/prpc/v1/Health` | -| External | `Worker`, closed | `/prpc` | `/prpc/Worker.Info` | +The scheme is uniform: `v0` names the frozen v0.5.11 surface, `v1` the current +one, on both listeners. + +| Listener | Version | Service | Mount | Example path | +|---|---|---|---|---| +| Internal socket | v1 | `DstackGuestV1` | `/v1` | `/v1/GetKey` | +| Internal socket | v0 | `DstackGuest`, frozen | `/v0` | `/v0/GetKey` | +| Internal socket | v0 alias | `DstackGuest`, frozen | `/` | `/GetKey` | +| External | v1 | `WorkerV1` | `/prpc/v1` | `/prpc/v1/Health` | +| External | v0 | `Worker`, frozen | `/prpc/v0` | `/prpc/v0/Info` | +| External | v0 alias | `Worker`, frozen | `/prpc` | `/prpc/Worker.Info` | + +The unversioned paths are compatibility aliases, kept so a pre-0.6 client keeps +working unchanged. They are additional mounts of the *same* handler, not a +parallel implementation, so they cannot drift from `/v0`. New code should say +which version it means. + +`Tappd` is unchanged and outside this scheme: it predates v0 and stays on its own +socket at `/prpc/`. The internal socket is `/var/run/dstack.sock`, reachable only by the application itself. The external listener is reachable by anyone who can route to the CVM. @@ -182,7 +194,7 @@ Joining with `:` or `/` is not sufficient here and is not what v1 does. ### The KDF ```text -salt = "RATLS" (5 bytes, ASCII, unchanged from v0) +salt = "dstack-guest-v1" (15 bytes, ASCII; v0 uses "RATLS") IKM = app root secp256k1 private key (32 bytes, `k256_key` from .appkeys.json) info = LP("dstack-guest-v1-key") || LP(algorithm) || LP(domain) L = 32 @@ -193,9 +205,23 @@ key = HKDF-SHA256(salt, IKM, info, L) (RFC 5869: extract, then expand) `algorithm` is bound as its canonical name, `secp256k1` or `ed25519`, never as the string the caller sent. -The primitive family is unchanged from v0, which used the same HKDF-SHA256 and the -same salt. What changed is the `info`: v0 passed the path alone, so the algorithm -did not participate and one 32-byte secret served both curves. +The primitive is unchanged from v0, which also used HKDF-SHA256 over the same +input key material. Two things changed. The `info` now binds the algorithm and a +version tag, where v0 passed the path alone, so the algorithm did not participate +and one 32-byte secret served both curves. And the salt is v1's own. + +The salt is what makes v1 a separate derivation tree rather than a +differently-labelled branch of the old one. Under a shared salt the two surfaces +would be separated only by their HKDF `info` -- and the legacy `info` is the +caller's `path` verbatim, so a caller that passed the v1 `info` byte string as a +v0 `path` would reproduce a v1 key exactly. Different salts close that by +construction, whatever either side puts in `info`. + +That collision was never a privilege boundary: both derivations serve the same +single-tenant application from the same root key, and that application may call +either surface. It is closed because a KDF whose separation depends on nobody +choosing an awkward input is one refactor away from not separating anything, and +the fix costs nothing on a surface with no deployed keys yet. The 32 output bytes are used directly: @@ -205,13 +231,6 @@ The 32 output bytes are used directly: two domains on one key. - **ed25519**: the RFC 8032 seed, from which the key expands as usual. -Because v1 and v0 share the HKDF salt, a caller that supplies a crafted `path` to -the unversioned `GetKey` can reproduce a v1 key. That is not a boundary. Both -derivations serve the same single-tenant application from the same root key, and -that application can call either surface. The domain separation here prevents -accidental reuse across algorithms and versions; it does not isolate a caller from -itself, and nothing in dstack's threat model asks it to. - ### Public key encoding | Algorithm | Encoding | Length | @@ -233,12 +252,12 @@ Generated with an app root key of | domain | algorithm | private key | public key | |---|---|---|---| -| `""` | secp256k1 | `463e877bc7322c1c09e567844b3101e88f353bfb33177c41cb13832cb67eef1c` | `03c45e036d19662802e628d9a712c07a9d9d64bee28e1754a72d75010860a789c2` | -| `""` | ed25519 | `a41f6458de9d11a43f79640b6ab2c62d02aceda7b16c5f159dc7ad69621c7eb0` | `7a5740fa9ab4791a232cc1fc7e73d5ff47ad41be2589a71b05f128dee4223f08` | -| `wallet` | secp256k1 | `c2b47271c2956c020eb471f3d6ec9a08bac4ce72d158078592c3bfd9db67808c` | `0375b11b6fabbe6e18b9bac26b082070dea76487ce512323870bc784a28ec5404b` | -| `wallet` | ed25519 | `fce7a47f848fc9f7a799a34e061f4d07d7e8ef3adda98d8fc36a5de3f5146cdf` | `5e8076b492634770e9b12dc8b136c9f5e3e8a86adc657040ba7a320296dcf6b0` | -| `a/b/c` | secp256k1 | `448f92c86abd72c1c35b7ff75b5c9971114694e7825518e5ed0d2101711527cd` | `03011845be1d30004c148ae49ef3a82585094f856a7c9bc3ed06edf959537a4eac` | -| `k\0:ey` | ed25519 | `3cfcf09543e94d23c87aec1e5774ca3803feb3dd8ee298507650409b20ebfdde` | `9ba2d94591b97db4f089fc187cfcb09a9cd55bdd553e066d80cc8cb2977bd841` | +| `""` | secp256k1 | `59f60584ce6fd2a3a31997256db9d77322463fc8a6b1520110401bcb1ee92387` | `0377c7fb050db181d392266a3cee9adb2901c6d665f11bac68be5457f577ba4908` | +| `""` | ed25519 | `b023493030669cf22e9cafa6a464d4cf3ae4edfe5474ec796710f21ea011946d` | `a3dc149fd5b765eab2eb7d3174fa939e39386898f10b15b7b146f6f1358ecf2a` | +| `wallet` | secp256k1 | `2580611f0f936abe59399a8ac4ed9964d0259bd34c88ea012ca42b32acbf9386` | `0369cecd3c8da88730f7d45875824c3e75f63a2d3da4be42f45671954daa2abb28` | +| `wallet` | ed25519 | `d76a703b08ebb074b809b9d6acf3d7c6663131273807717ce9d23bbadc2c644e` | `dade622d0fa1641e79b16e0b04e296be671f85f0aa6387b7d37e9d89f87494f5` | +| `a/b/c` | secp256k1 | `7f0973449298085d2d36a3b4c4d3243c100ba1981ffa885fe9e9dee883e69538` | `02e9b1a61b6d70aa9b241753828c316bf90e33e77b2e113f9ba75a8b6dc3cde5c1` | +| `k\0:ey` | ed25519 | `42da8bf0b479ed125c370e3b91f982735bf08ff592abbd586985affa43ee96a1` | `c833107822b003ff5675b33b90b151d4315c3ab9162b17d876e8dffde41abf9b` | The last row's domain is the five bytes `6b 00 3a 65 79`. It is there because a delimiter-joined encoding would mishandle it. @@ -282,14 +301,14 @@ A worked claim, for `domain = "wallet"` and `algorithm = "secp256k1"`: 00 00 00 19 "dstack-guest-v1-key-claim" 25 bytes 00 00 00 09 "secp256k1" 9 bytes 00 00 00 06 "wallet" 6 bytes -00 00 00 21 03 75 b1 1b ... 04 4b 33 bytes +00 00 00 21 03 69 ce cd ... bb 28 33 bytes ``` With the app root key from the test vector table, `link0` is ```text -96726db35263cb2a7067fc5ebb0d06621f7cab2d0f0aa15a83858dbf9ff7f12c -6e1c5640223fcca1a03ba5ccf09d353609db6481f9563add16e16a8ad8544c29 +af26d2f258d34580e7288bd83fc97bddc83769476d77823c4f76a3ad77a75149 +1a39ffc4ef3aa66cb0d008b8f6f199e6d57c1da9a92ba4cf10f23bf752b8cad0 00 ```