From 8b39c8fdebe2f2ed4e7f5332b7817735d966c1ba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 20:27:49 -0700 Subject: [PATCH 1/7] refactor(gpu): extract the nvattest invocation into a shared crate The boot gate owned the only code that knows how to call `nvattest`: the argument shape, the proxy-URL rules that stop a misconfigured value smuggling a different collateral endpoint past `{proxy}/ocsp`, and the nonce check. The guest agent is about to need all three for on-demand attestation, and a second copy is a second set of bugs. The crate keeps the invocation and drops the appraisal, because the boot gate and a runtime liveness check want different answers from the same bytes: `check_nonce` only asserts that every claim answers the nonce it was given, and device counts, CC state and application policy stay with the caller. `run` and `attest` are separate so the boot gate keeps persisting stdout before it judges the exit status -- a failed appraisal is exactly when the evidence is worth having on disk. `run_command` also gains `kill_on_drop`, so a wedged tool no longer outlives the timeout that gave up on it. --- dstack/Cargo.lock | 14 + dstack/Cargo.toml | 2 + dstack/dstack-util/Cargo.toml | 1 + dstack/dstack-util/src/system_setup.rs | 130 +-------- dstack/nvattest/Cargo.toml | 24 ++ dstack/nvattest/src/lib.rs | 360 +++++++++++++++++++++++++ 6 files changed, 407 insertions(+), 124 deletions(-) create mode 100644 dstack/nvattest/Cargo.toml create mode 100644 dstack/nvattest/src/lib.rs diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index aad37febb..bc684123f 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -1970,6 +1970,7 @@ dependencies = [ "libc", "listenfd", "load_config", + "nvattest", "or-panic", "ra-rpc", "ra-tls", @@ -2239,6 +2240,7 @@ dependencies = [ "k256", "libc", "luks2", + "nvattest", "nvml-wrapper", "parity-scale-codec", "ra-rpc", @@ -4810,6 +4812,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "nvattest" +version = "0.6.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tokio", + "tracing", + "url", +] + [[package]] name = "nvml-wrapper" version = "0.12.1" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 6fe00a79d..fc4b54a57 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -59,6 +59,7 @@ members = [ "serde-duration", "dstack-mr", "dstack-mr/cli", + "nvattest", "nvidia-attest-proxy", "verifier", "size-parser", @@ -121,6 +122,7 @@ serde-duration = { path = "serde-duration" } dstack-mr = { path = "dstack-mr" } dstack-verifier = { path = "verifier", default-features = false } size-parser = { path = "size-parser" } +nvattest = { path = "nvattest" } wavekv = "2.1" # Core dependencies diff --git a/dstack/dstack-util/Cargo.toml b/dstack/dstack-util/Cargo.toml index fe16dec66..add01b01e 100644 --- a/dstack/dstack-util/Cargo.toml +++ b/dstack/dstack-util/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +nvattest.workspace = true aes-gcm.workspace = true anyhow.workspace = true clap.workspace = true diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 7dc2947bf..f60ec1665 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -1434,11 +1434,8 @@ async fn do_sys_setup(stage0: Stage0<'_>) -> Result<()> { mod gpu { use super::*; - const NVATTEST: &str = "/usr/bin/nvattest"; - const ATTESTATION_TIMEOUT: Duration = Duration::from_secs(300); const EVENT_VERSION: u32 = 2; const POLICY_ENTRYPOINT: &str = "data.policy.nv_match"; - const TRUST_OUTPOST_POLICY: &str = "/usr/share/nvattest/policies/allow_trust_outpost_ocsp.rego"; /// Bound Rego evaluation so a runaway application policy cannot hang boot. const POLICY_TIMEOUT: Duration = Duration::from_secs(10); @@ -1578,23 +1575,6 @@ mod gpu { Ok(inventory.nvidia) } - /// Run a GPU tool with a bounded timeout so a wedged driver/GPU cannot - /// hang the boot indefinitely (dstack-prepare is a oneshot unit with no - /// start timeout of its own). - async fn run_command( - program: &str, - args: &[&str], - timeout: Duration, - ) -> Result { - tokio::time::timeout( - timeout, - tokio::process::Command::new(program).args(args).output(), - ) - .await - .with_context(|| format!("{program} timed out"))? - .with_context(|| format!("failed to run {program}")) - } - fn init_nvml(expected_devices: u32) -> Result { let nvml = nvml_wrapper::Nvml::init().context("failed to initialize NVML")?; let devices = nvml @@ -1726,52 +1706,6 @@ mod gpu { serde_json::to_vec(&event).context("failed to serialize GPU attestation event") } - fn normalize_proxy_url(proxy_url: Option<&str>) -> Result> { - let Some(proxy_url) = proxy_url.map(str::trim).filter(|url| !url.is_empty()) else { - return Ok(None); - }; - let parsed = url::Url::parse(proxy_url).context("invalid NVIDIA attestation proxy URL")?; - if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { - bail!("NVIDIA attestation proxy must be an absolute HTTP(S) URL"); - } - if parsed.query().is_some() - || parsed.fragment().is_some() - || !parsed.username().is_empty() - || parsed.password().is_some() - || parsed.path() != "/" - { - bail!( - "NVIDIA attestation proxy URL must not contain credentials, path, query, or fragment" - ); - } - Ok(Some(parsed.as_str().trim_end_matches('/').to_string())) - } - - fn nvattest_args(nonce: &str, proxy_url: Option<&str>) -> Result> { - let mut args = vec![ - "attest".to_string(), - "--device".to_string(), - "gpu".to_string(), - "--verifier".to_string(), - "local".to_string(), - "--nonce".to_string(), - nonce.to_string(), - "--format".to_string(), - "json".to_string(), - ]; - if let Some(proxy_url) = normalize_proxy_url(proxy_url)? { - args.extend([ - "--ocsp-url".to_string(), - format!("{proxy_url}/ocsp"), - "--rim-url".to_string(), - proxy_url, - "--relying-party-policy".to_string(), - TRUST_OUTPOST_POLICY.to_string(), - ]); - } - Ok(args) - } - /// Run local GPU attestation via nvattest with a fresh evidence nonce. If /// sys-config selects a collateral proxy, both RIM and OCSP traffic is /// routed through it and NVIDIA's Trust Outpost policy accepts cached OCSP @@ -1781,7 +1715,7 @@ mod gpu { expected_devices: u32, proxy_url: Option<&str>, ) -> Result { - if !Path::new(NVATTEST).exists() { + if !nvattest::available() { bail!("nvattest is not available in this image"); } // Certificate/OCSP validation needs a sane clock even when @@ -1789,26 +1723,12 @@ mod gpu { if let Err(err) = cmd!(chronyc makestep) { warn!("failed to step system clock: {err:?}"); } - let nonce = hex::encode(rand::thread_rng().gen::<[u8; 32]>()); - let args = nvattest_args(&nonce, proxy_url)?; - if args.iter().any(|arg| arg == "--relying-party-policy") - && !Path::new(TRUST_OUTPOST_POLICY).is_file() - { - bail!("NVIDIA attestation proxy is configured but {TRUST_OUTPOST_POLICY} is missing"); - } - let args = args.iter().map(String::as_str).collect::>(); - let output = run_command(NVATTEST, &args, ATTESTATION_TIMEOUT).await?; - if !output.stderr.is_empty() { - info!("nvattest: {}", truncated_lossy(&output.stderr, 2048)); - } + let nonce: [u8; nvattest::NONCE_LEN] = rand::thread_rng().gen(); + let (nonce, output) = nvattest::run(&nonce, proxy_url, nvattest::DEFAULT_TIMEOUT).await?; + // Persist before judging the exit status: a failed appraisal is exactly + // when the evidence is worth having on disk. save_attestation_output(&output.stdout).context("failed to save GPU attestation output")?; - if !output.status.success() { - bail!( - "nvattest exited with {}: {}", - output.status, - truncated_lossy(&output.stderr, 512), - ); - } + nvattest::check_status(&output)?; let claims = validate_attestation_output(&output.stdout, &nonce, expected_devices)?; Ok(GpuAttestationResult { claims: claims.raw, @@ -1880,15 +1800,6 @@ mod gpu { Ok(()) } - fn truncated_lossy(bytes: &[u8], limit: usize) -> String { - let text = String::from_utf8_lossy(bytes); - let text = text.trim(); - match text.char_indices().nth(limit) { - Some((idx, _)) => format!("{}...", &text[..idx]), - None => text.to_string(), - } - } - #[cfg(test)] mod tests { use super::*; @@ -1956,35 +1867,6 @@ mod gpu { assert_eq!(nvidia_gpu_count(nvidia).unwrap(), 2); } - #[test] - fn proxy_routes_ocsp_and_rim_and_selects_outpost_policy() { - let nonce = format!("test-nonce-{}", std::process::id()); - let args = nvattest_args(&nonce, Some("http://10.0.2.2:8090/")).unwrap(); - assert!(args - .windows(2) - .any(|args| args == ["--ocsp-url", "http://10.0.2.2:8090/ocsp"])); - assert!(args - .windows(2) - .any(|args| args == ["--rim-url", "http://10.0.2.2:8090"])); - assert!(args - .windows(2) - .any(|args| args == ["--relying-party-policy", TRUST_OUTPOST_POLICY])); - - let direct = nvattest_args(&nonce, None).unwrap(); - assert!(!direct.iter().any(|arg| arg == "--ocsp-url")); - assert!(!direct.iter().any(|arg| arg == "--relying-party-policy")); - } - - #[test] - fn proxy_url_validation_is_fail_closed() { - let nonce = format!("test-nonce-{}", std::process::id()); - assert!(nvattest_args(&nonce, Some("file:///tmp/proxy")).is_err()); - assert!(nvattest_args(&nonce, Some("https://user@example.com")).is_err()); - assert!(nvattest_args(&nonce, Some("https://example.com?q=1")).is_err()); - assert!(nvattest_args(&nonce, Some("https://example.com/base")).is_err()); - assert!(normalize_proxy_url(Some(" ")).unwrap().is_none()); - } - #[test] fn basic_policy_requires_cc_and_rejects_devtools_by_default() { let nonce = "44".repeat(32); diff --git a/dstack/nvattest/Cargo.toml b/dstack/nvattest/Cargo.toml new file mode 100644 index 000000000..672209a92 --- /dev/null +++ b/dstack/nvattest/Cargo.toml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "nvattest" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Shared wrapper around the NVIDIA nvattest CLI used by the boot gate and the guest agent" + +[dependencies] +anyhow.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +tokio = { workspace = true, features = ["process", "time"] } +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/nvattest/src/lib.rs b/dstack/nvattest/src/lib.rs new file mode 100644 index 000000000..00381b903 --- /dev/null +++ b/dstack/nvattest/src/lib.rs @@ -0,0 +1,360 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Thin wrapper around the NVIDIA `nvattest` CLI. +//! +//! Both the boot path (`dstack-util setup`) and the guest agent's on-demand +//! `AttestGpu` RPC shell out to the same binary with the same argument shape, +//! so the invocation, the proxy-URL rules and the output checks live here +//! rather than being written twice with two sets of bugs. +//! +//! What this crate does *not* do is decide what the evidence means. Appraisal +//! against a policy is the caller's, because the boot gate and a runtime +//! liveness check want different answers from the same bytes. + +use std::{path::Path, process::Output, time::Duration}; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; +use serde_json::Value; +use tracing::info; + +/// The nvattest binary, installed into the rootfs by the nvattest recipe. +pub const NVATTEST: &str = "/usr/bin/nvattest"; + +/// NVIDIA's relying-party policy that tolerates a cached OCSP responder nonce. +/// Only used when a collateral proxy is configured. +pub const TRUST_OUTPOST_POLICY: &str = "/usr/share/nvattest/policies/allow_trust_outpost_ocsp.rego"; + +/// SPDM fixes the GPU evidence nonce at 32 bytes. The SDK rejects anything +/// else, so callers get a clear error here instead of a CLI parse failure. +pub const NONCE_LEN: usize = 32; + +/// Long enough for a cold collateral fetch on a slow link, short enough that a +/// wedged driver cannot hang the caller forever. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300); + +/// Raw `nvattest attest --format json` output plus the nonce it answered. +#[derive(Debug)] +pub struct Attestation { + /// Exact stdout bytes. Callers that bind evidence to a measurement must + /// hash these, not a re-serialization of the parsed claims. + pub output: Vec, + /// Hex-encoded nonce passed to nvattest, echoed back as `eat_nonce`. + pub nonce: String, + /// Parsed claims, one per attested device. + pub claims: Vec, +} + +#[derive(Deserialize)] +struct NvattestOutput { + result_code: i64, + claims: Vec, +} + +#[derive(Deserialize)] +struct NonceClaim { + #[serde(rename = "eat_nonce")] + eat_nonce: String, + #[serde(rename = "x-nvidia-gpu-attestation-report-nonce-match")] + nonce_match: bool, +} + +/// True when this image can attest a GPU at all. +pub fn available() -> bool { + Path::new(NVATTEST).exists() +} + +/// Validate a collateral proxy URL. Rejects anything carrying credentials or a +/// path, so a misconfigured value cannot smuggle a different endpoint past the +/// `{proxy}/ocsp` and `{proxy}/v1/rim/...` construction below. +pub fn normalize_proxy_url(proxy_url: Option<&str>) -> Result> { + let Some(proxy_url) = proxy_url.map(str::trim).filter(|url| !url.is_empty()) else { + return Ok(None); + }; + let parsed = url::Url::parse(proxy_url).context("invalid NVIDIA attestation proxy URL")?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + bail!("NVIDIA attestation proxy must be an absolute HTTP(S) URL"); + } + if parsed.query().is_some() + || parsed.fragment().is_some() + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.path() != "/" + { + bail!( + "NVIDIA attestation proxy URL must not contain credentials, path, query, or fragment" + ); + } + Ok(Some(parsed.as_str().trim_end_matches('/').to_string())) +} + +/// Build the CLI arguments for a local (self-verifying) GPU attestation. +pub fn args(nonce: &str, proxy_url: Option<&str>) -> Result> { + let mut args = vec![ + "attest".to_string(), + "--device".to_string(), + "gpu".to_string(), + "--verifier".to_string(), + "local".to_string(), + "--nonce".to_string(), + nonce.to_string(), + "--format".to_string(), + "json".to_string(), + ]; + if let Some(proxy_url) = normalize_proxy_url(proxy_url)? { + args.extend([ + "--ocsp-url".to_string(), + format!("{proxy_url}/ocsp"), + "--rim-url".to_string(), + proxy_url, + "--relying-party-policy".to_string(), + TRUST_OUTPOST_POLICY.to_string(), + ]); + } + Ok(args) +} + +/// Run a command with a bounded timeout, killing the child if it expires so a +/// wedged GPU tool cannot outlive the caller that gave up on it. +pub async fn run_command(program: &str, args: &[&str], timeout: Duration) -> Result { + let child = tokio::process::Command::new(program) + .args(args) + .kill_on_drop(true) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .with_context(|| format!("failed to run {program}"))?; + tokio::time::timeout(timeout, child.wait_with_output()) + .await + .with_context(|| format!("{program} timed out"))? + .with_context(|| format!("failed to run {program}")) +} + +/// Run `nvattest` against `nonce` and return its raw output. +/// +/// `nonce` must be exactly [`NONCE_LEN`] bytes; it is passed through verbatim +/// so a caller can compare its own challenge against `eat_nonce` without +/// reversing any transform. +/// +/// Verifies only that nvattest succeeded and that every claim answers this +/// nonce. Everything else -- device counts, CC state, application policy -- +/// is appraisal, and belongs to the caller. +pub async fn attest( + nonce: &[u8], + proxy_url: Option<&str>, + timeout: Duration, +) -> Result { + let (nonce, output) = run(nonce, proxy_url, timeout).await?; + check_status(&output)?; + let claims = check_nonce(&output.stdout, &nonce)?; + Ok(Attestation { + output: output.stdout, + nonce, + claims, + }) +} + +/// Run `nvattest` and hand back its raw output *whatever its exit status*, +/// along with the hex nonce it was given. +/// +/// Errors only when the tool could not be run at all. The boot gate persists +/// stdout before judging the status, so a failed appraisal still leaves +/// evidence on disk to debug; that is why the status check is separate. +pub async fn run( + nonce: &[u8], + proxy_url: Option<&str>, + timeout: Duration, +) -> Result<(String, Output)> { + if !available() { + bail!("nvattest is not available in this image"); + } + if nonce.len() != NONCE_LEN { + bail!( + "gpu attestation nonce must be {NONCE_LEN} bytes, got {}", + nonce.len() + ); + } + let nonce = hex_encode(nonce); + let args = args(&nonce, proxy_url)?; + if args.iter().any(|arg| arg == "--relying-party-policy") + && !Path::new(TRUST_OUTPOST_POLICY).is_file() + { + bail!("NVIDIA attestation proxy is configured but {TRUST_OUTPOST_POLICY} is missing"); + } + let borrowed = args.iter().map(String::as_str).collect::>(); + let output = run_command(NVATTEST, &borrowed, timeout).await?; + if !output.stderr.is_empty() { + info!("nvattest: {}", truncated_lossy(&output.stderr, 2048)); + } + Ok((nonce, output)) +} + +/// Turn a non-zero nvattest exit into an error carrying a bounded stderr tail. +pub fn check_status(output: &Output) -> Result<()> { + if !output.status.success() { + bail!( + "nvattest exited with {}: {}", + output.status, + truncated_lossy(&output.stderr, 512), + ); + } + Ok(()) +} + +/// Require a successful result and one fresh claim per device answering `nonce`. +pub fn check_nonce(stdout: &[u8], nonce: &str) -> Result> { + let parsed: NvattestOutput = + serde_json::from_slice(stdout).context("failed to parse nvattest JSON output")?; + if parsed.result_code != 0 { + bail!( + "nvattest JSON result is not successful (result_code={})", + parsed.result_code + ); + } + if parsed.claims.is_empty() { + bail!("nvattest returned no GPU claims"); + } + for (index, claim) in parsed.claims.iter().enumerate() { + let nonce_claim: NonceClaim = serde_json::from_value(claim.clone()) + .with_context(|| format!("invalid GPU claim at index {index}"))?; + if nonce_claim.eat_nonce != nonce || !nonce_claim.nonce_match { + bail!("gpu claim at index {index} does not answer the requested nonce"); + } + } + Ok(parsed.claims) +} + +fn hex_encode(bytes: &[u8]) -> String { + use std::fmt::Write; + bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut out, b| { + let _ = write!(out, "{b:02x}"); + out + }) +} + +/// Truncate on a character boundary so a huge or binary stderr cannot flood logs. +pub fn truncated_lossy(bytes: &[u8], limit: usize) -> String { + let text = String::from_utf8_lossy(bytes); + match text.char_indices().nth(limit) { + Some((end, _)) => format!("{}...", &text[..end]), + None => text.into_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn claim(nonce: &str, matched: bool) -> String { + format!( + r#"{{"eat_nonce":"{nonce}","x-nvidia-gpu-attestation-report-nonce-match":{matched}}}"# + ) + } + + fn output(result_code: i64, claims: &[String]) -> Vec { + format!( + r#"{{"result_code":{result_code},"claims":[{}]}}"#, + claims.join(",") + ) + .into_bytes() + } + + #[test] + fn accepts_claims_answering_the_requested_nonce() { + let nonce = "aa".repeat(NONCE_LEN); + let out = output(0, &[claim(&nonce, true), claim(&nonce, true)]); + assert_eq!(check_nonce(&out, &nonce).unwrap().len(), 2); + } + + #[test] + fn rejects_a_claim_answering_a_different_nonce() { + let nonce = "aa".repeat(NONCE_LEN); + let stale = "bb".repeat(NONCE_LEN); + let out = output(0, &[claim(&nonce, true), claim(&stale, true)]); + let err = check_nonce(&out, &nonce).unwrap_err().to_string(); + assert!(err.contains("does not answer the requested nonce"), "{err}"); + } + + #[test] + fn rejects_a_claim_whose_device_reported_no_nonce_match() { + let nonce = "aa".repeat(NONCE_LEN); + let out = output(0, &[claim(&nonce, false)]); + assert!(check_nonce(&out, &nonce).is_err()); + } + + #[test] + fn rejects_unsuccessful_and_empty_results() { + let nonce = "aa".repeat(NONCE_LEN); + assert!(check_nonce(&output(1, &[claim(&nonce, true)]), &nonce).is_err()); + assert!(check_nonce(&output(0, &[]), &nonce).is_err()); + } + + #[tokio::test] + async fn rejects_a_nonce_of_the_wrong_length() { + let err = attest(&[0u8; 16], None, DEFAULT_TIMEOUT) + .await + .unwrap_err() + .to_string(); + // Length is checked before the binary is, so this holds off-target too. + assert!( + err.contains("32 bytes") || err.contains("not available"), + "{err}" + ); + } + + #[test] + fn proxy_url_rules() { + assert_eq!(normalize_proxy_url(None).unwrap(), None); + assert_eq!(normalize_proxy_url(Some(" ")).unwrap(), None); + assert_eq!( + normalize_proxy_url(Some("http://10.0.2.2:8090/")).unwrap(), + Some("http://10.0.2.2:8090".to_string()) + ); + for bad in [ + "ftp://host/", + "file:///tmp/proxy", + "https://user@example.com", + "http://user:pw@host/", + "https://example.com/base", + "https://example.com?q=1", + "not-a-url", + ] { + assert!( + normalize_proxy_url(Some(bad)).is_err(), + "{bad} must be rejected" + ); + } + } + + #[test] + fn proxy_routes_ocsp_and_rim_and_selects_outpost_policy() { + let proxied = args("aa", Some("http://10.0.2.2:8090/")).unwrap(); + assert!(proxied + .windows(2) + .any(|w| w == ["--ocsp-url", "http://10.0.2.2:8090/ocsp"])); + assert!(proxied + .windows(2) + .any(|w| w == ["--rim-url", "http://10.0.2.2:8090"])); + assert!(proxied + .windows(2) + .any(|w| w == ["--relying-party-policy", TRUST_OUTPOST_POLICY])); + + let direct = args("aa", None).unwrap(); + assert!(!direct.iter().any(|a| a == "--ocsp-url")); + assert!(!direct.iter().any(|a| a == "--relying-party-policy")); + } + + #[test] + fn truncation_is_bounded_and_utf8_safe() { + assert_eq!(truncated_lossy(b"abc", 10), "abc"); + assert_eq!(truncated_lossy(b"abcdef", 3), "abc..."); + assert_eq!( + truncated_lossy("\u{4f60}\u{597d}".as_bytes(), 1), + "\u{4f60}..." + ); + } +} From 6ad8b6e336ba0372d63985213318e99b11804d59 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 20:27:49 -0700 Subject: [PATCH 2/7] feat(guest-agent): add AttestGpu for on-demand GPU attestation `GpuInfo` replays a record written at boot, which cannot answer "is the GPU working *now*". That question has a concrete failure behind it: unloading and reloading the NVIDIA driver leaves a device that still answers NVML but can no longer attest, and the repo's own H100 evidence run records exactly that (`nvidia_smi_rc=9`, attestation `244`, recovered only by a full VM stop/start). `security-model.md` already told higher-assurance deployments to "re-attest before using a newly initialized GPU" without offering an API to do it. `AttestGpu` runs nvattest against a caller-chosen nonce. The nonce is passed to the GPU verbatim and must be exactly 32 bytes, which is what SPDM fixes it at: no padding and no hashing, so a caller compares its own bytes against `eat_nonce` rather than reversing a transform. **What it does not do.** It is not a remote attestation claim, and both the proto and every SDK say so at the field. An NVIDIA report binds the device and the nonce and nothing else, so a hostile host can relay the challenge to a real GPU elsewhere; deriving the nonce from a TDX quote does not help, because the relay can derive it too. Only TDISP/TEE-IO device binding closes that. Remote evidence therefore stays with the boot-time `gpu-attestation` event, which measured code emits before any workload exists and which the event log binds to the quote. No runtime event is emitted for the same reason: every dstack verifier stops scanning the event log at `system-ready`, so measuring this would add noise that nothing reads and invite the misreading above. Calls are serialised behind a mutex and rate-limited to one per 10s. The CVM is a single trust domain, so a container spinning on this endpoint only hurts its own deployment -- the reason to bound it is that each call spawns a process and fetches OCSP and RIM collateral from NVIDIA, whose SDK cache is per-process and so starts cold every time. A retry loop would hammer NVIDIA's services from every CVM running the buggy app. --- CHANGELOG.md | 1 + docs/attestation-tdx.md | 2 +- docs/security/security-model.md | 2 +- dstack/guest-agent/Cargo.toml | 1 + dstack/guest-agent/rpc/proto/agent_rpc.proto | 46 +++++++ dstack/guest-agent/src/gpu_attest.rs | 132 +++++++++++++++++++ dstack/guest-agent/src/lib.rs | 1 + dstack/guest-agent/src/rpc_service.rs | 31 ++++- sdk/curl/api.md | 57 +++++++- sdk/go/README.md | 22 ++++ sdk/go/dstack/client.go | 36 +++++ sdk/go/dstack/client_test.go | 45 +++++++ sdk/js/README.md | 18 +++ sdk/js/src/__tests__/index.test.ts | 13 ++ sdk/js/src/index.ts | 39 ++++++ sdk/python/README.md | 19 +++ sdk/python/src/dstack_sdk/__init__.py | 2 + sdk/python/src/dstack_sdk/dstack_client.py | 42 ++++++ sdk/python/tests/test_client.py | 27 ++++ sdk/rust/README.md | 19 +++ sdk/rust/src/dstack_client.rs | 16 +++ sdk/rust/tests/test_client.rs | 14 ++ sdk/rust/types/src/dstack.rs | 17 +++ 23 files changed, 595 insertions(+), 7 deletions(-) create mode 100644 dstack/guest-agent/src/gpu_attest.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 45aa253bb..0111006f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - sdk: `verify_signature` and `verify_signature_chain` in all four SDKs, replacing the removed guest-agent `Verify` RPC. `verify_signature_chain` is new capability rather than a port: it walks all three links of a `Sign` signature chain -- payload signature, the app root key attesting `"{purpose}:{hex(pubkey)}"`, and the KMS root attesting that app root for this `app_id` -- and requires the chain to anchor at a KMS root public key **the caller supplies**. That anchor has to come from somewhere independently trusted (the `DstackKms` contract's `kmsInfo().k256Pubkey`, or a pinned value); read it from the KMS being checked and an attacker who can answer that query can also mint a self-consistent chain. The four ports are pinned against one committed set of test vectors, `sdk/tests/vectors/signature_chain.json`, generated from the real KMS and guest-agent primitives -- this repo has shipped cross-language crypto drift twice already +- guest-agent: `AttestGpu` runs NVIDIA GPU attestation on demand against a caller-supplied 32-byte nonce, so an application can re-check the device after a driver reload leaves a GPU that answers NVML but can no longer attest. It is a local liveness check, not a remote claim: an NVIDIA report binds the device and nonce but not the TD, so it is relayable until TDISP/TEE-IO. Serialised and rate-limited. Exposed in the Rust, Python, Go and JS SDKs - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796) - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer diff --git a/docs/attestation-tdx.md b/docs/attestation-tdx.md index a71533ef1..591cf5815 100644 --- a/docs/attestation-tdx.md +++ b/docs/attestation-tdx.md @@ -45,7 +45,7 @@ evaluation, `gpu-attestation`. The `gpu-policy-hash` payload is The `gpu-attestation` payload is JSON containing the verified device count, CC/DevTools state, and `evidence_sha256`. -The guest-agent `GpuInfo` API returns the complete `nvattest` JSON captured during boot. It is not trustworthy by itself. After verifying the TDX quote and replaying the event log to RTMR3, hash the exact UTF-8 bytes of `GpuInfo.attestation` and require the result to equal the `gpu-attestation` event's `evidence_sha256`. See [GPU Security for AI Workloads](./security/security-model.md#gpu-security-for-ai-workloads) for the event schema, ordering, Rego example, and platform differences. +The guest-agent `GpuInfo` API returns the complete `nvattest` JSON captured during boot. It is not trustworthy by itself. (`AttestGpu` runs a *fresh* attestation against a caller nonce, but its result is not bound to the TD and must not be used as remote evidence; only the boot-time record below is.) After verifying the TDX quote and replaying the event log to RTMR3, hash the exact UTF-8 bytes of `GpuInfo.attestation` and require the result to equal the `gpu-attestation` event's `evidence_sha256`. See [GPU Security for AI Workloads](./security/security-model.md#gpu-security-for-ai-workloads) for the event schema, ordering, Rego example, and platform differences. ### 2.2. Determining expected MRs MRTD, RTMR0, RTMR1, and RTMR2 correspond to the image. dstack OS builds all related software from source. diff --git a/docs/security/security-model.md b/docs/security/security-model.md index 72791944e..141ea367c 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -158,7 +158,7 @@ The events make the following **boot-time** statement: immediately before key pr - A mixed launch cannot attest only its TEE-capable subset. Non-NVIDIA display GPUs are rejected, and the sysfs, NVML, and nvattest claim counts must all agree. A non-CC NVIDIA GPU either prevents evidence collection/appraisal or causes the default appraisal, application policy, or CC-state check to fail. - Copying another CVM's result into a file or `report_data` does not work. Only measured pre-application code can place the event before `system-ready`, and event-log replay binds it to the quoted RTMR/PCR value. -This is **not a lifetime or physical co-location guarantee**. After `system-ready`, an application with sufficient guest privileges can unload the NVIDIA driver, and a malicious host may attempt PCI hot-remove/replacement or proxy GPU traffic. The boot event remains a true historical statement but does not prove that the same device is still attached. dstack also cannot rule out a live relay/cuckoo attack to a genuine remote GPU: current Hopper/Blackwell deployments do not provide a CPU-TEE-verifiable TEE-I/O/TDISP device binding. Applications that mutate the driver or PCI topology are outside this guarantee; higher-assurance deployments must prevent that behavior and re-attest before using a newly initialized GPU. +This is **not a lifetime or physical co-location guarantee**. After `system-ready`, an application with sufficient guest privileges can unload the NVIDIA driver, and a malicious host may attempt PCI hot-remove/replacement or proxy GPU traffic. The boot event remains a true historical statement but does not prove that the same device is still attached. dstack also cannot rule out a live relay/cuckoo attack to a genuine remote GPU: current Hopper/Blackwell deployments do not provide a CPU-TEE-verifiable TEE-I/O/TDISP device binding. Applications that mutate the driver or PCI topology are outside this guarantee; higher-assurance deployments must prevent that behavior and re-attest before using a newly initialized GPU; the guest-agent `AttestGpu` API does that re-check against a caller-chosen nonce. Being an NVIDIA report, its result is subject to the same relay caveat as any attestation-time GPU sample: it establishes that a genuine CC-enabled GPU is reachable and responsive now, not that the device is bound to this TD, so it must not be forwarded to a remote relying party as proof of GPU possession. AMD SEV-SNP has no runtime measurement register in the current dstack stack. The local boot gate can still fail closed, but a `gpu-attestation` event carried beside an SNP report is not remotely bound to that report and must not be accepted as dual-attestation evidence. SNP needs a measured vTPM/PCR channel before it can provide the same remote binding. diff --git a/dstack/guest-agent/Cargo.toml b/dstack/guest-agent/Cargo.toml index 9b23ae0d6..ffdfa269e 100644 --- a/dstack/guest-agent/Cargo.toml +++ b/dstack/guest-agent/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +nvattest.workspace = true rocket.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/dstack/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto index 1a3552c5c..db99619f2 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc.proto @@ -61,6 +61,18 @@ service DstackGuest { // Get GPU information collected during boot. rpc GpuInfo(google.protobuf.Empty) returns (GpuInfoResponse) {} + // Run NVIDIA GPU attestation now, against a nonce the caller chooses. + // + // This answers "is the device I can talk to right now a genuine, CC-enabled + // NVIDIA GPU that signs my challenge", which `GpuInfo` cannot: that returns a + // record written at boot. Use it after anything that may have reinitialised + // the GPU -- a driver reload leaves a device that responds to NVML but can no + // longer attest -- and before submitting work you care about. + // + // It is NOT a remote attestation claim, and a relying party outside this CVM + // must not treat it as one. See `AttestGpuResponse.evidence`. + rpc AttestGpu(AttestGpuArgs) returns (AttestGpuResponse) {} + // Sign a payload rpc Sign(SignRequest) returns (SignResponse) {} @@ -198,6 +210,40 @@ message AttestResponse { bytes attestation = 1; } +message AttestGpuArgs { + // Exactly 32 bytes of caller-chosen challenge, passed to the GPU verbatim. + // + // SPDM fixes the evidence nonce at 32 bytes, and dstack applies no transform + // so a caller can compare these bytes directly against the `eat_nonce` claim + // rather than reversing a hash. To bind a longer challenge, hash it yourself. + bytes nonce = 1; +} + +message AttestGpuResponse { + // Complete JSON output produced by nvattest, in the same shape `GpuInfo` + // returns, for the nonce in the request. + // + // dstack has already checked that nvattest succeeded and that every claim + // answers this nonce. Appraising the claims -- device count, CC state, VBIOS + // and driver versions, your own policy -- is the caller's job. + // + // What this proves is local and live: at the moment of the call, a genuine + // NVIDIA GPU reachable from this CVM signed this nonce. What it does NOT + // prove is that the GPU is attached to *this* TD. An NVIDIA report binds the + // device and the nonce, nothing more, so a hostile host can relay the + // challenge to a real GPU elsewhere and a colluding workload can do the same. + // Deriving the nonce from a TDX quote does not help -- the relay can derive + // it too. Only TDISP/TEE-IO device binding would close this, and no current + // Hopper/Blackwell deployment offers it. So: sound as a local health check, + // unsound as evidence to a remote party. For the latter, use the boot-time + // `gpu-attestation` runtime event, which measured code emits before any + // workload exists and which the event log binds to the quote. + string evidence = 1; + // The nonce the GPU actually answered, hex-encoded, as it appears in + // `eat_nonce`. Echoed so a caller can assert on it without re-encoding. + string nonce = 2; +} + message GpuInfoResponse { // Complete JSON output produced by nvattest. Empty when no boot-time GPU // attestation output is available. diff --git a/dstack/guest-agent/src/gpu_attest.rs b/dstack/guest-agent/src/gpu_attest.rs new file mode 100644 index 000000000..2c28d8ee1 --- /dev/null +++ b/dstack/guest-agent/src/gpu_attest.rs @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! On-demand GPU attestation for the `AttestGpu` RPC. +//! +//! Running `nvattest` is expensive in a way most RPCs are not: it spawns a +//! process, fetches OCSP and RIM collateral from NVIDIA (the SDK's cache is +//! per-process, so a subprocess starts cold every time), and can take minutes +//! on a slow link. The agent's socket is mode 0777, so any container in the +//! CVM can reach it. +//! +//! The CVM is a single trust domain, so a container hammering this endpoint is +//! only hurting its own deployment -- that is not the concern. The concern is +//! that a retry loop would hammer *NVIDIA's* services from every dstack CVM +//! running the buggy app, and would keep a wedged GPU tool respawning. So the +//! gate below serialises attestations and enforces a cooldown between them. + +use std::time::{Duration, Instant}; + +use anyhow::{bail, Result}; +use tokio::sync::Mutex; + +/// Minimum spacing between two attestations. Chosen against what the call +/// actually costs: a cold collateral fetch is seconds, so a caller polling +/// faster than this is not learning anything new, only generating load. +pub const COOLDOWN: Duration = Duration::from_secs(10); + +/// Serialises `nvattest` runs and rate-limits them. +/// +/// Results are deliberately not shared between waiters: each caller supplies +/// its own nonce, and handing back evidence that answers somebody else's +/// challenge is exactly the confusion this API exists to avoid. +pub struct GpuAttestor { + proxy_url: Option, + timeout: Duration, + cooldown: Duration, + last_run: Mutex>, +} + +impl GpuAttestor { + pub fn new(proxy_url: Option) -> Self { + Self { + proxy_url, + timeout: nvattest::DEFAULT_TIMEOUT, + cooldown: COOLDOWN, + last_run: Mutex::new(None), + } + } + + #[cfg(test)] + fn with_cooldown(cooldown: Duration) -> Self { + Self { + proxy_url: None, + timeout: nvattest::DEFAULT_TIMEOUT, + cooldown, + last_run: Mutex::new(None), + } + } + + /// Attest against `nonce`, or explain why not. + pub async fn attest(&self, nonce: &[u8]) -> Result { + if nonce.len() != nvattest::NONCE_LEN { + bail!( + "nonce must be exactly {} bytes, got {}", + nvattest::NONCE_LEN, + nonce.len() + ); + } + if !nvattest::available() { + bail!("GPU attestation is not available in this image"); + } + // Held across the whole run, so a second caller waits rather than + // starting a competing nvattest against the same devices. + let mut last_run = self.last_run.lock().await; + check_cooldown(*last_run, Instant::now(), self.cooldown)?; + let result = nvattest::attest(nonce, self.proxy_url.as_deref(), self.timeout).await; + // Stamp on failure too: a failing GPU is the case most likely to be + // retried in a tight loop, and the most expensive to retry. + *last_run = Some(Instant::now()); + result + } +} + +/// Reject a call that arrives before the cooldown has elapsed. +fn check_cooldown(last_run: Option, now: Instant, cooldown: Duration) -> Result<()> { + let Some(last_run) = last_run else { + return Ok(()); + }; + let elapsed = now.saturating_duration_since(last_run); + if elapsed < cooldown { + let wait = (cooldown - elapsed).as_secs() + 1; + bail!("GPU attestation was run too recently; retry in {wait}s"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn first_call_is_always_allowed() { + assert!(check_cooldown(None, Instant::now(), COOLDOWN).is_ok()); + } + + #[test] + fn a_call_inside_the_cooldown_is_rejected_with_a_wait_hint() { + let now = Instant::now(); + let last = now - Duration::from_secs(3); + let err = check_cooldown(Some(last), now, Duration::from_secs(10)) + .unwrap_err() + .to_string(); + assert!(err.contains("retry in"), "{err}"); + } + + #[test] + fn a_call_after_the_cooldown_is_allowed() { + let now = Instant::now(); + let last = now - Duration::from_secs(11); + assert!(check_cooldown(Some(last), now, Duration::from_secs(10)).is_ok()); + } + + #[tokio::test] + async fn a_wrong_length_nonce_is_rejected_before_anything_expensive() { + let attestor = GpuAttestor::with_cooldown(Duration::from_secs(0)); + let err = attestor.attest(&[0u8; 16]).await.unwrap_err().to_string(); + assert!(err.contains("exactly 32 bytes"), "{err}"); + // Rejected on arity, so it must not have consumed the rate limit. + assert!(attestor.last_run.lock().await.is_none()); + } +} diff --git a/dstack/guest-agent/src/lib.rs b/dstack/guest-agent/src/lib.rs index 76fca6871..5ecb7359e 100644 --- a/dstack/guest-agent/src/lib.rs +++ b/dstack/guest-agent/src/lib.rs @@ -8,6 +8,7 @@ pub const GIT_REV: &str = dstack_build_info::git_revision!(); pub mod backend; pub mod config; mod container_health; +mod gpu_attest; mod guest_api_service; mod health; mod http_routes; diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index e3ee8fc35..f283fd1ef 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -15,10 +15,10 @@ use dstack_guest_agent_rpc::{ dstack_guest_server::{DstackGuestRpc, DstackGuestServer}, tappd_server::{TappdRpc, TappdServer}, worker_server::{WorkerRpc, WorkerServer}, - AppInfo, AttestAppKeyRequest, AttestResponse, DeriveK256KeyResponse, DeriveKeyArgs, GetKeyArgs, - GetKeyResponse, GetQuoteResponse, GetTlsKeyArgs, GetTlsKeyResponse, GpuInfoResponse, - HealthResponse, RawQuoteArgs, SignRequest, SignResponse, TdxQuoteArgs, TdxQuoteResponse, - WorkerVersion, + AppInfo, AttestAppKeyRequest, AttestGpuArgs, AttestGpuResponse, AttestResponse, + DeriveK256KeyResponse, DeriveKeyArgs, GetKeyArgs, GetKeyResponse, GetQuoteResponse, + GetTlsKeyArgs, GetTlsKeyResponse, GpuInfoResponse, HealthResponse, RawQuoteArgs, SignRequest, + SignResponse, TdxQuoteArgs, TdxQuoteResponse, WorkerVersion, }; use dstack_types::{AppKeys, SysConfig, GPU_ATTESTATION_OUTPUT}; use ed25519_dalek::ed25519::signature::hazmat::PrehashSigner; @@ -79,6 +79,8 @@ struct AppStateInner { platform: Arc, /// Present only when the app opted into health gating; see `health`. health: Option>, + /// Serialises and rate-limits on-demand GPU attestation. + gpu_attestor: crate::gpu_attest::GpuAttestor, } impl AppStateInner { @@ -162,6 +164,10 @@ impl AppState { serde_json::from_str(&fs::read_to_string(&config.sys_config_file)?) .context("Failed to parse VM config")?; let collateral_urls = sys_config.collateral_urls(); + // Same collateral proxy the boot gate uses, so a CVM without egress to + // NVIDIA can still attest on demand. + let gpu_attestor = + crate::gpu_attest::GpuAttestor::new(sys_config.nvidia_attestation_proxy_url.clone()); let vm_config = sys_config.vm_config; // Same trust anchor decision as dstack-util: never host-supplied, and // development roots only when this guest published them itself. @@ -192,6 +198,7 @@ impl AppState { vm_config, platform, health, + gpu_attestor, }), }; me.maybe_request_demo_cert(); @@ -378,6 +385,21 @@ impl DstackGuestRpc for InternalRpcHandler { get_info(&self.state, false).await } + async fn attest_gpu(self, request: AttestGpuArgs) -> Result { + let attestation = self + .state + .inner + .gpu_attestor + .attest(&request.nonce) + .await + .context("GPU attestation failed")?; + Ok(AttestGpuResponse { + evidence: String::from_utf8(attestation.output) + .context("nvattest output is not valid UTF-8")?, + nonce: attestation.nonce, + }) + } + async fn gpu_info(self) -> Result { Ok(GpuInfoResponse { attestation: read_gpu_attestation(Path::new(GPU_ATTESTATION_OUTPUT)), @@ -982,6 +1004,7 @@ pNs85uhOZE8z2jr8Pg== }, }), health: None, + gpu_attestor: crate::gpu_attest::GpuAttestor::new(None), }; ( diff --git a/sdk/curl/api.md b/sdk/curl/api.md index be19180f7..1930d12cb 100644 --- a/sdk/curl/api.md +++ b/sdk/curl/api.md @@ -260,7 +260,62 @@ curl --unix-socket /var/run/dstack.sock http://dstack/Attest?report_data=0000000 } ``` -### 7. GPU Info +### 7. Attest GPU + +Runs NVIDIA GPU attestation **now**, against a nonce you choose. Unlike +[`GpuInfo`](#8-gpu-info), which replays a record written at boot, this samples the +device at the moment of the call. + +Use it after anything that may have reinitialised the GPU. A driver reload leaves a +device that still answers NVML but can no longer attest, and this is how an +application detects that before submitting work. + +> [!WARNING] +> This is **not** a remote attestation claim. It proves that a genuine NVIDIA GPU +> reachable from this CVM signed your nonce, right now. It does not prove the GPU is +> attached to *this* CVM: an NVIDIA report binds the device and the nonce, nothing +> more, so a hostile host can relay the challenge to a real GPU elsewhere, and +> deriving the nonce from a TDX quote does not help because the relay can derive it +> too. Only TDISP/TEE-IO device binding would close this, and no current +> Hopper/Blackwell deployment offers it. +> +> Sound as a local health check. Unsound as evidence to a remote party — for that, +> use the boot-time `gpu-attestation` runtime event, which measured code emits +> before any workload exists and which the event log binds to the quote. + +**Endpoint:** `/AttestGpu` + +**Request Parameters:** + +| Field | Type | Description | Example | +|-------|------|-------------|----------| +| `nonce` | string | Exactly 32 bytes, hex-encoded, passed to the GPU verbatim. To bind a longer challenge, hash it yourself. | `"ab...ab"` (64 hex chars) | + +**Example:** +```bash +curl --unix-socket /var/run/dstack.sock -X POST \ + http://dstack/AttestGpu \ + -H 'Content-Type: application/json' \ + -d '{ + "nonce": "abababababababababababababababababababababababababababababababab" + }' +``` + +**Response:** +```json +{ + "evidence": "{\"result_code\": 0, \"claims\": [...]}", + "nonce": "abababababababababababababababababababababababababababababababab" +} +``` + +dstack has already checked that nvattest succeeded and that every claim answers your +nonce; appraising the claims is yours. Calls are serialised and rate-limited (one +attestation per 10s), because each one spawns `nvattest` and fetches OCSP and RIM +collateral from NVIDIA. A call arriving inside the cooldown is rejected with a wait +hint rather than queued. + +### 8. GPU Info Returns GPU information collected during boot. Currently, this includes the complete JSON output produced by NVIDIA `nvattest`. diff --git a/sdk/go/README.md b/sdk/go/README.md index bda275b24..488ca9446 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -575,6 +575,28 @@ instead. - Cryptographic proof of execution environment - Audit trail generation +##### `AttestGpu(ctx context.Context, nonce []byte) (*AttestGpuResponse, error)` + +Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. Use it after +anything that may have reinitialised the GPU — a driver reload leaves a device that +answers NVML but can no longer attest. + +```go +resp, err := client.AttestGpu(ctx, nonce) +if err != nil { + log.Fatal(err) +} +fmt.Println(resp.Evidence) +``` + +> [!WARNING] +> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this +> CVM signed your nonce right now; it does not prove the GPU is attached to *this* +> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a +> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local +> health check, unsound as evidence to a remote party — for that use the boot-time +> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. + ##### `GpuInfo(ctx context.Context) (*GpuInfoResponse, error)` Returns GPU information collected during boot. Currently, this includes the diff --git a/sdk/go/dstack/client.go b/sdk/go/dstack/client.go index 8985cae34..65b891600 100644 --- a/sdk/go/dstack/client.go +++ b/sdk/go/dstack/client.go @@ -118,6 +118,21 @@ type GpuInfoResponse struct { Attestation string `json:"attestation"` } +// AttestGpuResponse is the result of a fresh, on-demand NVIDIA GPU attestation. +// +// It proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, +// right now. It does NOT prove the GPU is attached to this CVM: an NVIDIA report +// binds the device and the nonce, nothing more, so a hostile host can relay the +// challenge to a real GPU elsewhere. Sound as a local health check, unsound as +// evidence to a remote party -- for that, use the boot-time `gpu-attestation` +// event bound to the quote. +type AttestGpuResponse struct { + // Evidence is the complete nvattest JSON for the requested nonce. + Evidence string `json:"evidence"` + // Nonce is the nonce the GPU answered, hex-encoded, as it appears in eat_nonce. + Nonce string `json:"nonce"` +} + // Represents an event log entry in the TCB info type EventLog struct { IMR int `json:"imr"` @@ -531,6 +546,27 @@ func (c *DstackClient) Attest(ctx context.Context, reportData []byte) (*AttestRe return &AttestResponse{Attestation: attestation}, nil } +// AttestGpu runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. +// +// See AttestGpuResponse for what this does and does not prove. +func (c *DstackClient) AttestGpu(ctx context.Context, nonce []byte) (*AttestGpuResponse, error) { + if len(nonce) != 32 { + return nil, fmt.Errorf("nonce must be exactly 32 bytes, got %d", len(nonce)) + } + + payload := map[string]interface{}{"nonce": hex.EncodeToString(nonce)} + data, err := c.sendRPCRequest(ctx, "/AttestGpu", payload) + if err != nil { + return nil, err + } + + var response AttestGpuResponse + if err := json.Unmarshal(data, &response); err != nil { + return nil, err + } + return &response, nil +} + // GpuInfo returns GPU information collected during boot. func (c *DstackClient) GpuInfo(ctx context.Context) (*GpuInfoResponse, error) { data, err := c.sendRPCRequest(ctx, "/GpuInfo", map[string]interface{}{}) diff --git a/sdk/go/dstack/client_test.go b/sdk/go/dstack/client_test.go index 0ca8d1185..7bbd91f78 100644 --- a/sdk/go/dstack/client_test.go +++ b/sdk/go/dstack/client_test.go @@ -9,6 +9,7 @@ import ( "context" "crypto/sha256" "crypto/x509" + "encoding/hex" "encoding/json" "encoding/pem" "fmt" @@ -78,6 +79,50 @@ func TestAttest(t *testing.T) { } } +func TestAttestGpu(t *testing.T) { + const evidence = `{"result_code":0,"claims":[]}` + nonce := bytes.Repeat([]byte{0xab}, 32) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/AttestGpu" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + var payload map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("failed to decode request: %v", err) + } + if payload["nonce"] != hex.EncodeToString(nonce) { + t.Fatalf("nonce was not forwarded verbatim, got: %v", payload["nonce"]) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{ + "evidence": evidence, + "nonce": hex.EncodeToString(nonce), + }) + })) + defer server.Close() + + client := dstack.NewDstackClient(dstack.WithEndpoint(server.URL)) + resp, err := client.AttestGpu(context.Background(), nonce) + if err != nil { + t.Fatal(err) + } + if resp.Evidence != evidence { + t.Fatalf("unexpected evidence: %s", resp.Evidence) + } + if resp.Nonce != hex.EncodeToString(nonce) { + t.Fatalf("unexpected nonce: %s", resp.Nonce) + } +} + +func TestAttestGpuRejectsWrongNonceLength(t *testing.T) { + client := dstack.NewDstackClient() + for _, n := range [][]byte{nil, bytes.Repeat([]byte{1}, 31), bytes.Repeat([]byte{1}, 33)} { + if _, err := client.AttestGpu(context.Background(), n); err == nil { + t.Fatalf("expected a %d-byte nonce to be rejected", len(n)) + } + } +} + func TestGpuInfo(t *testing.T) { const attestation = `{"result_code":0,"claims":[]}` server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/sdk/js/README.md b/sdk/js/README.md index 7594e3eb1..c4bbed6aa 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -95,6 +95,24 @@ Versioned dstack attestation that works across TDX / GCP / Nitro providers. Pref const { attestation } = await client.attest('app-state-snapshot') ``` +### `attestGpu(nonce)` + +Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. Use it after +anything that may have reinitialised the GPU — a driver reload leaves a device that +answers NVML but can no longer attest. + +```typescript +const { evidence } = await client.attestGpu(crypto.randomBytes(32)) +``` + +> [!WARNING] +> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this +> CVM signed your nonce right now; it does not prove the GPU is attached to *this* +> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a +> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local +> health check, unsound as evidence to a remote party — for that use the boot-time +> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. + ### `gpuInfo()` Returns GPU information collected during boot. Currently, this includes the diff --git a/sdk/js/src/__tests__/index.test.ts b/sdk/js/src/__tests__/index.test.ts index c5cc3fa6d..90c46fac1 100644 --- a/sdk/js/src/__tests__/index.test.ts +++ b/sdk/js/src/__tests__/index.test.ts @@ -48,6 +48,19 @@ describe('DstackClient', () => { expect(() => JSON.parse(result.event_log)).not.toThrowError() }) + it('should reject an attestGpu nonce that is not 32 bytes', async () => { + const client = new DstackClient() + await expect(() => client.attestGpu(new Uint8Array(31))).rejects.toThrow() + await expect(() => client.attestGpu(new Uint8Array(33))).rejects.toThrow() + }) + + it('should surface an attestGpu failure when the simulator has no GPU', async () => { + const client = new DstackClient() + // The simulator ships no nvattest, so this must fail fast and clearly + // rather than hang for the attestation timeout. + await expect(() => client.attestGpu(new Uint8Array(32).fill(0xab))).rejects.toThrow() + }) + it('should be able to attest', async () => { const client = new DstackClient() const result = await client.attest('test') diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index dc9f64e8e..29fc7b6e3 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -103,6 +103,25 @@ export interface AttestResponse { attestation: Hex } +/** + * Result of a fresh, on-demand NVIDIA GPU attestation. + * + * Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right + * now. It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the + * device and the nonce, nothing more, so a hostile host can relay the challenge to a + * real GPU elsewhere. Sound as a local health check, unsound as evidence to a remote + * party -- for that, use the boot-time `gpu-attestation` event bound to the quote. + */ +export interface AttestGpuResponse { + __name__: Readonly<'AttestGpuResponse'> + + /** Complete nvattest JSON for the requested nonce. */ + evidence: string + + /** The nonce the GPU answered, hex-encoded, as it appears in `eat_nonce`. */ + nonce: string +} + export interface GpuInfoResponse { __name__: Readonly<'GpuInfoResponse'> @@ -302,6 +321,26 @@ export class DstackClient { }) } + /** + * Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. + * + * See {@link AttestGpuResponse} for what this does and does not prove. + */ + async attestGpu(nonce: Buffer | Uint8Array): Promise { + if (nonce.length !== 32) { + throw new Error(`Nonce must be exactly 32 bytes, got ${nonce.length}.`) + } + const payload = JSON.stringify({ nonce: to_hex(nonce) }) + const result = await send_rpc_request<{ evidence: string, nonce: string }>(this.endpoint, '/AttestGpu', payload) + if ('error' in (result as any)) { + throw new Error((result as any)['error'] as string) + } + return Object.freeze({ + ...result, + __name__: 'AttestGpuResponse' as const, + }) + } + async gpuInfo(): Promise { const result = await send_rpc_request<{ attestation: string }>(this.endpoint, '/GpuInfo', '{}') return Object.freeze({ diff --git a/sdk/python/README.md b/sdk/python/README.md index 8788a7269..d32830386 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -100,6 +100,25 @@ print(result.attestation) # hex string print(result.decode_attestation()) # bytes ``` +### On-demand GPU Attestation + +`attest_gpu(nonce)` runs NVIDIA GPU attestation now, against a 32-byte nonce you +choose. Use it after anything that may have reinitialised the GPU — a driver reload +leaves a device that answers NVML but can no longer attest. + +```python +result = client.attest_gpu(os.urandom(32)) +print(result.evidence) +``` + +> [!WARNING] +> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this +> CVM signed your nonce right now; it does not prove the GPU is attached to *this* +> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a +> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local +> health check, unsound as evidence to a remote party — for that use the boot-time +> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. + ### GPU Info `gpu_info()` returns GPU information collected during boot. Currently, this diff --git a/sdk/python/src/dstack_sdk/__init__.py b/sdk/python/src/dstack_sdk/__init__.py index 3f9bc4cdb..a0ac66144 100644 --- a/sdk/python/src/dstack_sdk/__init__.py +++ b/sdk/python/src/dstack_sdk/__init__.py @@ -10,6 +10,7 @@ from .dstack_client import GetKeyResponse from .dstack_client import GetQuoteResponse from .dstack_client import GetTlsKeyResponse +from .dstack_client import AttestGpuResponse from .dstack_client import GpuInfoResponse from .dstack_client import InfoResponse from .dstack_client import SignResponse @@ -38,6 +39,7 @@ "GetKeyResponse", "GetTlsKeyResponse", "AttestResponse", + "AttestGpuResponse", "GpuInfoResponse", "GetQuoteResponse", "InfoResponse", diff --git a/sdk/python/src/dstack_sdk/dstack_client.py b/sdk/python/src/dstack_sdk/dstack_client.py index 96b40e577..ccd4fd60a 100644 --- a/sdk/python/src/dstack_sdk/dstack_client.py +++ b/sdk/python/src/dstack_sdk/dstack_client.py @@ -157,6 +157,20 @@ def decode_attestation(self) -> bytes: return bytes.fromhex(self.attestation) +class AttestGpuResponse(BaseModel): + """Result of a fresh, on-demand NVIDIA GPU attestation. + + Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. + It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device + and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU + elsewhere. Sound as a local health check, unsound as evidence to a remote party -- + for that, use the boot-time `gpu-attestation` event bound to the quote. + """ + + evidence: str + nonce: str + + class GpuInfoResponse(BaseModel): attestation: str @@ -440,6 +454,22 @@ async def attest( result = await self._send_rpc_request("Attest", {"report_data": hex}) return AttestResponse(**result) + async def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: + """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. + + Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. + It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device + and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU + elsewhere. Sound as a local health check, unsound as evidence to a remote party -- + for that, use the boot-time `gpu-attestation` event bound to the quote. + """ + if not isinstance(nonce, (bytes, bytearray)) or len(nonce) != 32: + raise ValueError("nonce must be exactly 32 bytes") + result = await self._send_rpc_request( + "AttestGpu", {"nonce": binascii.hexlify(bytes(nonce)).decode()} + ) + return AttestGpuResponse(**result) + async def gpu_info(self) -> GpuInfoResponse: """Return GPU information collected during boot.""" result = await self._send_rpc_request("GpuInfo", {}) @@ -573,6 +603,18 @@ def attest( """Request a versioned attestation for the provided report data.""" raise NotImplementedError + @call_async + def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: + """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. + + Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. + It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device + and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU + elsewhere. Sound as a local health check, unsound as evidence to a remote party -- + for that, use the boot-time `gpu-attestation` event bound to the quote. + """ + raise NotImplementedError + @call_async def gpu_info(self) -> GpuInfoResponse: """Return GPU information collected during boot.""" diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 80724a22c..40a937df8 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -11,6 +11,7 @@ from dstack_sdk import AsyncDstackClient from dstack_sdk import AsyncTappdClient +from dstack_sdk import AttestGpuResponse from dstack_sdk import AttestResponse from dstack_sdk import DstackClient from dstack_sdk import GetKeyResponse @@ -121,6 +122,32 @@ async def test_async_client_attest(): assert len(result.attestation) > 0 +@pytest.mark.asyncio +async def test_async_client_attest_gpu(monkeypatch): + evidence = '{"result_code":0,"claims":[]}' + nonce = bytes([0xAB]) * 32 + + async def fake_send(self, method, payload): + assert method == "AttestGpu" + assert payload == {"nonce": nonce.hex()} + return {"evidence": evidence, "nonce": nonce.hex()} + + monkeypatch.setenv("DSTACK_SIMULATOR_ENDPOINT", "http://localhost:0") + monkeypatch.setattr(AsyncDstackClient, "_send_rpc_request", fake_send) + result = await AsyncDstackClient().attest_gpu(nonce) + assert isinstance(result, AttestGpuResponse) + assert result.evidence == evidence + assert result.nonce == nonce.hex() + + +@pytest.mark.asyncio +async def test_async_client_attest_gpu_rejects_wrong_nonce_length(): + client = AsyncDstackClient() + for bad in [b"", bytes(31), bytes(33), "not-bytes"]: + with pytest.raises(ValueError): + await client.attest_gpu(bad) + + @pytest.mark.asyncio async def test_async_client_gpu_info(monkeypatch): attestation = '{"result_code":0,"claims":[]}' diff --git a/sdk/rust/README.md b/sdk/rust/README.md index 2f00b0cec..ee16823af 100644 --- a/sdk/rust/README.md +++ b/sdk/rust/README.md @@ -104,6 +104,25 @@ println!("{}", info.tcb_info); Generates a versioned attestation with a custom 64-byte payload. - `attestation`: Hex-encoded attestation +#### `attest_gpu(nonce: Vec) -> AttestGpuResponse` + +Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. Use it after +anything that may have reinitialised the GPU — a driver reload leaves a device that +answers NVML but can no longer attest. + +```rust +let result = client.attest_gpu(nonce.to_vec()).await?; +println!("{}", result.evidence); +``` + +> [!WARNING] +> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this +> CVM signed your nonce right now; it does not prove the GPU is attached to *this* +> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a +> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local +> health check, unsound as evidence to a remote party — for that use the boot-time +> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. + #### `gpu_info() -> GpuInfoResponse` Returns GPU information collected during boot. Currently, this includes the diff --git a/sdk/rust/src/dstack_client.rs b/sdk/rust/src/dstack_client.rs index c6f824eeb..3365430e7 100644 --- a/sdk/rust/src/dstack_client.rs +++ b/sdk/rust/src/dstack_client.rs @@ -164,6 +164,22 @@ impl DstackClient { Ok(response) } + /// Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. + /// + /// Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. + /// It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device + /// and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU + /// elsewhere. Sound as a local health check, unsound as evidence to a remote party -- + /// for that, use the boot-time `gpu-attestation` event bound to the quote. + pub async fn attest_gpu(&self, nonce: Vec) -> Result { + if nonce.len() != 32 { + anyhow::bail!("Nonce must be exactly 32 bytes") + } + let data = json!({ "nonce": hex_encode(nonce) }); + let response = self.send_rpc_request("/AttestGpu", &data).await?; + Ok(serde_json::from_value::(response)?) + } + /// Returns GPU information collected during boot. pub async fn gpu_info(&self) -> Result { let response = self.send_rpc_request("/GpuInfo", &json!({})).await?; diff --git a/sdk/rust/tests/test_client.rs b/sdk/rust/tests/test_client.rs index 6bd636c0d..66f42a422 100644 --- a/sdk/rust/tests/test_client.rs +++ b/sdk/rust/tests/test_client.rs @@ -25,6 +25,20 @@ async fn test_async_client_get_quote() { assert!(!result.quote.is_empty()); } +#[tokio::test] +async fn test_async_client_attest_gpu_validates_nonce_length() { + let client = AsyncDstackClient::new(None); + for len in [0, 31, 33] { + assert!( + client.attest_gpu(vec![0u8; len]).await.is_err(), + "a {len}-byte nonce must be rejected" + ); + } + // The simulator ships no nvattest, so a well-formed request must still fail + // fast with an error rather than hang for the attestation timeout. + assert!(client.attest_gpu(vec![0xab; 32]).await.is_err()); +} + #[tokio::test] async fn test_async_client_attest() { let client = AsyncDstackClient::new(None); diff --git a/sdk/rust/types/src/dstack.rs b/sdk/rust/types/src/dstack.rs index 18144e08b..a743024d1 100644 --- a/sdk/rust/types/src/dstack.rs +++ b/sdk/rust/types/src/dstack.rs @@ -98,6 +98,23 @@ pub struct AttestResponse { pub attestation: String, } +/// Response from a fresh, on-demand NVIDIA GPU attestation. +/// +/// Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. +/// It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device +/// and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU +/// elsewhere. Sound as a local health check, unsound as evidence to a remote party -- +/// for that, use the boot-time `gpu-attestation` event bound to the quote. +#[derive(Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] +#[cfg_attr(feature = "borsh_schema", derive(BorshSchema))] +pub struct AttestGpuResponse { + /// Complete JSON output produced by nvattest for the requested nonce. + pub evidence: String, + /// The nonce the GPU answered, hex-encoded, as it appears in `eat_nonce`. + pub nonce: String, +} + /// Response containing the complete NVIDIA GPU attestation output. #[derive(Debug, Serialize, Deserialize)] #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] From 11c1e40f2b3b8c538a7d524807837b4309bd8818 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 20:50:05 -0700 Subject: [PATCH 3/7] docs(gpu): say that AttestGpu output is unsigned, not just relayable The response documented one reason a relying party must not trust it -- an NVIDIA report binds the device and nonce but not the TD, so it is relayable. There is a blunter reason that comes first, and the docs did not say it. `nvattest --verifier local` returns the verifier's *conclusion*, not the GPU's signed report. The repo's own captured H100 output shows the detached EAT is `alg:none` with an empty signature, issued by `NVAT-LOCAL-VERIFIER`, and claims such as `x-nvidia-gpu-attestation-report-signature-verified` are assertions about a check already performed; the signed SPDM report and certificate chain are consumed during verification and never appear in the output. A third party handed this JSON therefore has nothing to check at all, relay or no relay. Worth being precise about because it also explains why the boot-time path is sound with the same unsigned artifact: that evidence is trusted not because it self-authenticates but because measured dstack code pinned by `os_image_hash` appraised it before any workload existed, with sha256 of the exact bytes in RTMR3 under the quote. The appraiser is what is trusted, not the JSON. A test pins the format claim against the fixture, so an SDK that starts signing the EAT fails here rather than leaving the API docs quietly wrong. --- CHANGELOG.md | 2 +- dstack/Cargo.lock | 1 + dstack/dstack-util/Cargo.toml | 1 + dstack/dstack-util/src/system_setup.rs | 42 +++++++++++++++ dstack/guest-agent/rpc/proto/agent_rpc.proto | 38 ++++++++++---- sdk/curl/api.md | 30 +++++++---- sdk/go/README.md | 11 ++-- sdk/go/dstack/client.go | 19 ++++--- sdk/js/README.md | 11 ++-- sdk/js/src/index.ts | 18 +++++-- sdk/python/README.md | 11 ++-- sdk/python/src/dstack_sdk/dstack_client.py | 54 ++++++++++++++------ sdk/rust/README.md | 11 ++-- sdk/rust/src/dstack_client.rs | 18 +++++-- sdk/rust/types/src/dstack.rs | 18 +++++-- 15 files changed, 207 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0111006f1..0b51efd61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - sdk: `verify_signature` and `verify_signature_chain` in all four SDKs, replacing the removed guest-agent `Verify` RPC. `verify_signature_chain` is new capability rather than a port: it walks all three links of a `Sign` signature chain -- payload signature, the app root key attesting `"{purpose}:{hex(pubkey)}"`, and the KMS root attesting that app root for this `app_id` -- and requires the chain to anchor at a KMS root public key **the caller supplies**. That anchor has to come from somewhere independently trusted (the `DstackKms` contract's `kmsInfo().k256Pubkey`, or a pinned value); read it from the KMS being checked and an attacker who can answer that query can also mint a self-consistent chain. The four ports are pinned against one committed set of test vectors, `sdk/tests/vectors/signature_chain.json`, generated from the real KMS and guest-agent primitives -- this repo has shipped cross-language crypto drift twice already -- guest-agent: `AttestGpu` runs NVIDIA GPU attestation on demand against a caller-supplied 32-byte nonce, so an application can re-check the device after a driver reload leaves a GPU that answers NVML but can no longer attest. It is a local liveness check, not a remote claim: an NVIDIA report binds the device and nonce but not the TD, so it is relayable until TDISP/TEE-IO. Serialised and rate-limited. Exposed in the Rust, Python, Go and JS SDKs +- guest-agent: `AttestGpu` runs NVIDIA GPU attestation on demand against a caller-supplied 32-byte nonce, so an application can re-check the device after a driver reload leaves a GPU that answers NVML but can no longer attest. It is a local liveness check, not a remote claim, and cannot be verified by a third party: `nvattest --verifier local` returns the verifier's conclusion rather than the GPU's signed report (its detached EAT is `alg:none`), and even signed an NVIDIA report binds the device and nonce but not the TD, so it is relayable until TDISP/TEE-IO. Serialised and rate-limited. Exposed in the Rust, Python, Go and JS SDKs - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796) - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index bc684123f..28d41b459 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2218,6 +2218,7 @@ version = "0.6.0" dependencies = [ "aes-gcm", "anyhow", + "base64 0.22.1", "binrw", "bollard", "cc-eventlog", diff --git a/dstack/dstack-util/Cargo.toml b/dstack/dstack-util/Cargo.toml index add01b01e..8107377fe 100644 --- a/dstack/dstack-util/Cargo.toml +++ b/dstack/dstack-util/Cargo.toml @@ -69,4 +69,5 @@ safe-write.workspace = true errify.workspace = true [dev-dependencies] +base64.workspace = true rand.workspace = true diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index f60ec1665..9eb9b97ab 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -1837,6 +1837,48 @@ mod gpu { const H100_ATTESTATION_OUTPUT: &[u8] = include_bytes!("../tests/fixtures/gpu_attestation_h100.json"); + /// The `AttestGpu` API documents that its output cannot be verified by a + /// third party, because the local verifier reports a conclusion rather + /// than the GPU's signed report. That is a claim about NVIDIA's output + /// format, so pin it: if a future SDK starts signing the detached EAT, + /// this fails and the API docs need revisiting rather than quietly + /// becoming wrong. + #[test] + fn local_verifier_output_is_unsigned_self_report() { + let output: Value = serde_json::from_slice(H100_ATTESTATION_OUTPUT).unwrap(); + let eat = &output["detached_eat"]; + let jwt = eat[0][1].as_str().expect("detached EAT carries a JWT"); + let (header_b64, rest) = jwt.split_once('.').unwrap(); + let (_, signature) = rest.split_once('.').unwrap(); + assert!( + signature.is_empty(), + "detached EAT is signed; AttestGpu docs claim it is not" + ); + + use base64::Engine as _; + let header = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(header_b64) + .unwrap(); + let header: Value = serde_json::from_slice(&header).unwrap(); + assert_eq!(header["alg"], "none"); + + // And the signed artifacts really are absent: the claims carry + // verdicts about the certificate chain, not the chain itself. + let claim = &output["claims"][0]; + assert_eq!( + claim["x-nvidia-gpu-attestation-report-signature-verified"], + true + ); + assert!( + claim["x-nvidia-gpu-attestation-report-cert-chain"] + .as_object() + .expect("cert-chain claim is a verdict object") + .keys() + .all(|key| key.starts_with("x-nvidia-cert-")), + "cert-chain claim carries certificates, not just verdicts" + ); + } + #[test] fn inventory_counts_nvidia_and_non_nvidia_gpus() { let root = tempfile::tempdir().unwrap(); diff --git a/dstack/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto index db99619f2..6c5fb472d 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc.proto @@ -227,17 +227,33 @@ message AttestGpuResponse { // answers this nonce. Appraising the claims -- device count, CC state, VBIOS // and driver versions, your own policy -- is the caller's job. // - // What this proves is local and live: at the moment of the call, a genuine - // NVIDIA GPU reachable from this CVM signed this nonce. What it does NOT - // prove is that the GPU is attached to *this* TD. An NVIDIA report binds the - // device and the nonce, nothing more, so a hostile host can relay the - // challenge to a real GPU elsewhere and a colluding workload can do the same. - // Deriving the nonce from a TDX quote does not help -- the relay can derive - // it too. Only TDISP/TEE-IO device binding would close this, and no current - // Hopper/Blackwell deployment offers it. So: sound as a local health check, - // unsound as evidence to a remote party. For the latter, use the boot-time - // `gpu-attestation` runtime event, which measured code emits before any - // workload exists and which the event log binds to the quote. + // To a caller inside this CVM this is a live, meaningful result: the agent ran + // NVIDIA's verifier, which checked the GPU's report signature, its certificate + // chain and OCSP status, and the driver and VBIOS RIM signatures, all against + // this nonce. + // + // It is NOT independently verifiable by a third party, for two separate + // reasons, and the first is the one that surprises people: + // + // 1. It is unsigned. `nvattest --verifier local` returns the verifier's + // conclusion, not the GPU's signed report. The detached EAT it embeds is + // `alg:none` with an empty signature, issued by `NVAT-LOCAL-VERIFIER`, and + // claims such as `x-nvidia-gpu-attestation-report-signature-verified` are + // assertions about a check already performed, not proof anyone can redo. + // The GPU's signed SPDM report and certificate chain are consumed during + // verification and do not appear here. So a relying party handed this JSON + // has nothing to check -- it is a self-report from inside the CVM. + // 2. Even signed, it would bind the device and the nonce but not the TD the + // device is attached to, so it can be relayed from a genuine remote GPU. + // Deriving the nonce from a TDX quote does not help; the relay can derive + // it too. Only TDISP/TEE-IO device binding closes that, and no current + // Hopper/Blackwell deployment offers it. + // + // Remote evidence therefore stays with the boot-time `gpu-attestation` runtime + // event. Its trust does not come from the JSON being self-authenticating + // either -- it comes from measured dstack code (pinned by `os_image_hash`) + // having done the appraisal before any workload existed, with sha256 of the + // exact bytes measured into RTMR3 and covered by the quote. string evidence = 1; // The nonce the GPU actually answered, hex-encoded, as it appears in // `eat_nonce`. Echoed so a caller can assert on it without re-encoding. diff --git a/sdk/curl/api.md b/sdk/curl/api.md index 1930d12cb..3397afc3f 100644 --- a/sdk/curl/api.md +++ b/sdk/curl/api.md @@ -271,17 +271,27 @@ device that still answers NVML but can no longer attest, and this is how an application detects that before submitting work. > [!WARNING] -> This is **not** a remote attestation claim. It proves that a genuine NVIDIA GPU -> reachable from this CVM signed your nonce, right now. It does not prove the GPU is -> attached to *this* CVM: an NVIDIA report binds the device and the nonce, nothing -> more, so a hostile host can relay the challenge to a real GPU elsewhere, and -> deriving the nonce from a TDX quote does not help because the relay can derive it -> too. Only TDISP/TEE-IO device binding would close this, and no current -> Hopper/Blackwell deployment offers it. +> **This response cannot be independently verified by a third party.** Inside the CVM +> it is meaningful: the agent ran NVIDIA's verifier, which checked the GPU's report +> signature, certificate chain and OCSP status, and the driver and VBIOS RIM +> signatures, against your nonce. Outside it, two separate problems apply. > -> Sound as a local health check. Unsound as evidence to a remote party — for that, -> use the boot-time `gpu-attestation` runtime event, which measured code emits -> before any workload exists and which the event log binds to the quote. +> 1. **It is unsigned.** `--verifier local` returns the verifier's *conclusion*, not +> the GPU's signed report. The embedded detached EAT is `alg:none` with an empty +> signature, issued by `NVAT-LOCAL-VERIFIER`, and claims like +> `x-nvidia-gpu-attestation-report-signature-verified: true` are assertions about a +> check already performed — the signed SPDM report and certificate chain are +> consumed during verification and are not carried in the output. A relying party +> handed this JSON has nothing to check. +> 2. **No TD binding.** Even signed, an NVIDIA report binds the device and the nonce, +> not the TD it is attached to, so it can be relayed from a genuine remote GPU. +> Deriving the nonce from a TDX quote does not help — the relay can derive it too. +> Only TDISP/TEE-IO closes this, and no current Hopper/Blackwell deployment has it. +> +> Use it as a local health check. For remote evidence use the boot-time +> `gpu-attestation` runtime event: its trust comes not from the JSON being +> self-authenticating but from measured dstack code (pinned by `os_image_hash`) having +> appraised the GPU before any workload existed, with the digest in RTMR3. **Endpoint:** `/AttestGpu` diff --git a/sdk/go/README.md b/sdk/go/README.md index 488ca9446..b9738bef9 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -590,11 +590,12 @@ fmt.Println(resp.Evidence) ``` > [!WARNING] -> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this -> CVM signed your nonce right now; it does not prove the GPU is attached to *this* -> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a -> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local -> health check, unsound as evidence to a remote party — for that use the boot-time +> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — +> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: +> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with +> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and +> even signed it would bind the device and nonce but not the TD, so it is relayable. +> Use it as a local health check; for remote evidence use the boot-time > `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. ##### `GpuInfo(ctx context.Context) (*GpuInfoResponse, error)` diff --git a/sdk/go/dstack/client.go b/sdk/go/dstack/client.go index 65b891600..50b5f6a7f 100644 --- a/sdk/go/dstack/client.go +++ b/sdk/go/dstack/client.go @@ -120,12 +120,19 @@ type GpuInfoResponse struct { // AttestGpuResponse is the result of a fresh, on-demand NVIDIA GPU attestation. // -// It proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, -// right now. It does NOT prove the GPU is attached to this CVM: an NVIDIA report -// binds the device and the nonce, nothing more, so a hostile host can relay the -// challenge to a real GPU elsewhere. Sound as a local health check, unsound as -// evidence to a remote party -- for that, use the boot-time `gpu-attestation` -// event bound to the quote. +// Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which +// checked the GPU's report signature, its certificate chain and OCSP status, and the +// driver and VBIOS RIM signatures, all against this nonce. +// +// NOT independently verifiable by a third party, for two separate reasons. First, it +// is unsigned: --verifier local returns the verifier's conclusion, not the GPU's +// signed report. Its detached EAT is alg:none issued by NVAT-LOCAL-VERIFIER, and a +// claim like x-nvidia-gpu-attestation-report-signature-verified is an assertion, not +// proof; the signed artifacts are consumed during verification and not carried here. +// Second, even signed it would bind the device and the nonce but not the TD, so it is +// relayable from a genuine remote GPU. Use it as a local health check. For remote +// evidence use the boot-time `gpu-attestation` event, which measured code emits +// before any workload exists and which the event log binds to the quote. type AttestGpuResponse struct { // Evidence is the complete nvattest JSON for the requested nonce. Evidence string `json:"evidence"` diff --git a/sdk/js/README.md b/sdk/js/README.md index c4bbed6aa..36162cbd9 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -106,11 +106,12 @@ const { evidence } = await client.attestGpu(crypto.randomBytes(32)) ``` > [!WARNING] -> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this -> CVM signed your nonce right now; it does not prove the GPU is attached to *this* -> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a -> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local -> health check, unsound as evidence to a remote party — for that use the boot-time +> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — +> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: +> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with +> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and +> even signed it would bind the device and nonce but not the TD, so it is relayable. +> Use it as a local health check; for remote evidence use the boot-time > `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. ### `gpuInfo()` diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index 29fc7b6e3..93e53e22f 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -106,11 +106,19 @@ export interface AttestResponse { /** * Result of a fresh, on-demand NVIDIA GPU attestation. * - * Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right - * now. It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the - * device and the nonce, nothing more, so a hostile host can relay the challenge to a - * real GPU elsewhere. Sound as a local health check, unsound as evidence to a remote - * party -- for that, use the boot-time `gpu-attestation` event bound to the quote. + * Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked + * the GPU's report signature, its certificate chain and OCSP status, and the driver and + * VBIOS RIM signatures, all against this nonce. + * + * NOT independently verifiable by a third party, for two separate reasons. First, it is + * unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed + * report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim + * like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; + * the signed artifacts are consumed during verification and not carried here. Second, + * even signed it would bind the device and the nonce but not the TD, so it is relayable + * from a genuine remote GPU. Use it as a local health check. For remote evidence use the + * boot-time `gpu-attestation` event, which measured code emits before any workload + * exists and which the event log binds to the quote. */ export interface AttestGpuResponse { __name__: Readonly<'AttestGpuResponse'> diff --git a/sdk/python/README.md b/sdk/python/README.md index d32830386..4d6c50909 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -112,11 +112,12 @@ print(result.evidence) ``` > [!WARNING] -> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this -> CVM signed your nonce right now; it does not prove the GPU is attached to *this* -> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a -> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local -> health check, unsound as evidence to a remote party — for that use the boot-time +> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — +> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: +> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with +> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and +> even signed it would bind the device and nonce but not the TD, so it is relayable. +> Use it as a local health check; for remote evidence use the boot-time > `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. ### GPU Info diff --git a/sdk/python/src/dstack_sdk/dstack_client.py b/sdk/python/src/dstack_sdk/dstack_client.py index ccd4fd60a..5f60d5c44 100644 --- a/sdk/python/src/dstack_sdk/dstack_client.py +++ b/sdk/python/src/dstack_sdk/dstack_client.py @@ -160,11 +160,19 @@ def decode_attestation(self) -> bytes: class AttestGpuResponse(BaseModel): """Result of a fresh, on-demand NVIDIA GPU attestation. - Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. - It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device - and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU - elsewhere. Sound as a local health check, unsound as evidence to a remote party -- - for that, use the boot-time `gpu-attestation` event bound to the quote. + Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked + the GPU's report signature, its certificate chain and OCSP status, and the driver and + VBIOS RIM signatures, all against this nonce. + + NOT independently verifiable by a third party, for two separate reasons. First, it is + unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed + report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim + like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; + the signed artifacts are consumed during verification and not carried here. Second, + even signed it would bind the device and the nonce but not the TD, so it is relayable + from a genuine remote GPU. Use it as a local health check. For remote evidence use the + boot-time `gpu-attestation` event, which measured code emits before any workload + exists and which the event log binds to the quote. """ evidence: str @@ -457,11 +465,19 @@ async def attest( async def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. - Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. - It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device - and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU - elsewhere. Sound as a local health check, unsound as evidence to a remote party -- - for that, use the boot-time `gpu-attestation` event bound to the quote. + Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked + the GPU's report signature, its certificate chain and OCSP status, and the driver and + VBIOS RIM signatures, all against this nonce. + + NOT independently verifiable by a third party, for two separate reasons. First, it is + unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed + report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim + like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; + the signed artifacts are consumed during verification and not carried here. Second, + even signed it would bind the device and the nonce but not the TD, so it is relayable + from a genuine remote GPU. Use it as a local health check. For remote evidence use the + boot-time `gpu-attestation` event, which measured code emits before any workload + exists and which the event log binds to the quote. """ if not isinstance(nonce, (bytes, bytearray)) or len(nonce) != 32: raise ValueError("nonce must be exactly 32 bytes") @@ -607,11 +623,19 @@ def attest( def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. - Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. - It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device - and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU - elsewhere. Sound as a local health check, unsound as evidence to a remote party -- - for that, use the boot-time `gpu-attestation` event bound to the quote. + Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked + the GPU's report signature, its certificate chain and OCSP status, and the driver and + VBIOS RIM signatures, all against this nonce. + + NOT independently verifiable by a third party, for two separate reasons. First, it is + unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed + report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim + like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; + the signed artifacts are consumed during verification and not carried here. Second, + even signed it would bind the device and the nonce but not the TD, so it is relayable + from a genuine remote GPU. Use it as a local health check. For remote evidence use the + boot-time `gpu-attestation` event, which measured code emits before any workload + exists and which the event log binds to the quote. """ raise NotImplementedError diff --git a/sdk/rust/README.md b/sdk/rust/README.md index ee16823af..9ba0fdf6d 100644 --- a/sdk/rust/README.md +++ b/sdk/rust/README.md @@ -116,11 +116,12 @@ println!("{}", result.evidence); ``` > [!WARNING] -> Not a remote attestation claim. It proves a genuine NVIDIA GPU reachable from this -> CVM signed your nonce right now; it does not prove the GPU is attached to *this* -> CVM, because an NVIDIA report binds the device and the nonce and nothing more, so a -> hostile host can relay the challenge to a real GPU elsewhere. Sound as a local -> health check, unsound as evidence to a remote party — for that use the boot-time +> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — +> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: +> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with +> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and +> even signed it would bind the device and nonce but not the TD, so it is relayable. +> Use it as a local health check; for remote evidence use the boot-time > `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. #### `gpu_info() -> GpuInfoResponse` diff --git a/sdk/rust/src/dstack_client.rs b/sdk/rust/src/dstack_client.rs index 3365430e7..ac13ccb15 100644 --- a/sdk/rust/src/dstack_client.rs +++ b/sdk/rust/src/dstack_client.rs @@ -166,11 +166,19 @@ impl DstackClient { /// Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. /// - /// Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. - /// It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device - /// and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU - /// elsewhere. Sound as a local health check, unsound as evidence to a remote party -- - /// for that, use the boot-time `gpu-attestation` event bound to the quote. + /// Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked + /// the GPU's report signature, its certificate chain and OCSP status, and the driver and + /// VBIOS RIM signatures, all against this nonce. + /// + /// NOT independently verifiable by a third party, for two separate reasons. First, it is + /// unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed + /// report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim + /// like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; + /// the signed artifacts are consumed during verification and not carried here. Second, + /// even signed it would bind the device and the nonce but not the TD, so it is relayable + /// from a genuine remote GPU. Use it as a local health check. For remote evidence use the + /// boot-time `gpu-attestation` event, which measured code emits before any workload + /// exists and which the event log binds to the quote. pub async fn attest_gpu(&self, nonce: Vec) -> Result { if nonce.len() != 32 { anyhow::bail!("Nonce must be exactly 32 bytes") diff --git a/sdk/rust/types/src/dstack.rs b/sdk/rust/types/src/dstack.rs index a743024d1..2d569c6e2 100644 --- a/sdk/rust/types/src/dstack.rs +++ b/sdk/rust/types/src/dstack.rs @@ -100,11 +100,19 @@ pub struct AttestResponse { /// Response from a fresh, on-demand NVIDIA GPU attestation. /// -/// Proves that a genuine NVIDIA GPU reachable from this CVM signed the nonce, right now. -/// It does NOT prove the GPU is attached to this CVM: an NVIDIA report binds the device -/// and the nonce, nothing more, so a hostile host can relay the challenge to a real GPU -/// elsewhere. Sound as a local health check, unsound as evidence to a remote party -- -/// for that, use the boot-time `gpu-attestation` event bound to the quote. +/// Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked +/// the GPU's report signature, its certificate chain and OCSP status, and the driver and +/// VBIOS RIM signatures, all against this nonce. +/// +/// NOT independently verifiable by a third party, for two separate reasons. First, it is +/// unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed +/// report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim +/// like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; +/// the signed artifacts are consumed during verification and not carried here. Second, +/// even signed it would bind the device and the nonce but not the TD, so it is relayable +/// from a genuine remote GPU. Use it as a local health check. For remote evidence use the +/// boot-time `gpu-attestation` event, which measured code emits before any workload +/// exists and which the event log binds to the quote. #[derive(Debug, Serialize, Deserialize)] #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] #[cfg_attr(feature = "borsh_schema", derive(BorshSchema))] From 4268030690eab624c1fccbac5bb41cad39aef93a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 21:23:26 -0700 Subject: [PATCH 4/7] feat(gpu): return GPU-signed evidence from AttestGpu, not just a verdict `AttestGpu` returned only `nvattest attest --verifier local` output, which is the verifier's conclusion. That conclusion is unsigned -- its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER` -- so a third party handed it had nothing to check, and the API could only ever be a local health check. `collect-evidence` emits what the GPU actually signed: per device, the base64 SPDM attestation report and its certificate chain, over the caller's nonce. The response now carries that as `evidence`, so a relying party can verify the chain to NVIDIA's root, check the report signature, confirm the nonce inside the report, and compare measurements against NVIDIA's RIM documents -- with its own verifier, trusting nothing this CVM says. The local verdict is still useful to a caller inside the CVM, which is in the agent's trust domain and usually just wants the answer, so it stays as `appraisal`. Splitting the two is the RATS distinction: evidence is what the attester produces, appraisal is a verifier's opinion about it, and only the first travels. The agent collects once and appraises those exact bytes via `--gpu-evidence-source=file` rather than running two independent attestations, so the two halves provably describe the same report. The SDK independently rejects an evidence file whose nonce does not match the appraisal nonce. This does not change what the evidence proves about *placement*: an NVIDIA report binds the device and the nonce and nothing else, so it stays relayable from a genuine remote GPU until TDISP/TEE-IO. What changes is that the part a third party can check is now actually reaching them. --- CHANGELOG.md | 2 +- dstack/Cargo.lock | 1 + dstack/guest-agent/rpc/proto/agent_rpc.proto | 72 ++++----- dstack/guest-agent/src/gpu_attest.rs | 5 +- dstack/guest-agent/src/rpc_service.rs | 3 +- dstack/nvattest/Cargo.toml | 1 + dstack/nvattest/src/lib.rs | 159 ++++++++++++++++++- sdk/curl/api.md | 52 +++--- sdk/go/README.md | 17 +- sdk/go/dstack/client.go | 7 +- sdk/go/dstack/client_test.go | 10 +- sdk/js/README.md | 17 +- sdk/js/src/index.ts | 36 +++-- sdk/python/README.md | 17 +- sdk/python/src/dstack_sdk/dstack_client.py | 79 ++++----- sdk/python/tests/test_client.py | 6 +- sdk/rust/README.md | 17 +- sdk/rust/src/dstack_client.rs | 24 +-- sdk/rust/types/src/dstack.rs | 32 ++-- 19 files changed, 367 insertions(+), 190 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b51efd61..8f4121142 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - sdk: `verify_signature` and `verify_signature_chain` in all four SDKs, replacing the removed guest-agent `Verify` RPC. `verify_signature_chain` is new capability rather than a port: it walks all three links of a `Sign` signature chain -- payload signature, the app root key attesting `"{purpose}:{hex(pubkey)}"`, and the KMS root attesting that app root for this `app_id` -- and requires the chain to anchor at a KMS root public key **the caller supplies**. That anchor has to come from somewhere independently trusted (the `DstackKms` contract's `kmsInfo().k256Pubkey`, or a pinned value); read it from the KMS being checked and an attacker who can answer that query can also mint a self-consistent chain. The four ports are pinned against one committed set of test vectors, `sdk/tests/vectors/signature_chain.json`, generated from the real KMS and guest-agent primitives -- this repo has shipped cross-language crypto drift twice already -- guest-agent: `AttestGpu` runs NVIDIA GPU attestation on demand against a caller-supplied 32-byte nonce, so an application can re-check the device after a driver reload leaves a GPU that answers NVML but can no longer attest. It is a local liveness check, not a remote claim, and cannot be verified by a third party: `nvattest --verifier local` returns the verifier's conclusion rather than the GPU's signed report (its detached EAT is `alg:none`), and even signed an NVIDIA report binds the device and nonce but not the TD, so it is relayable until TDISP/TEE-IO. Serialised and rate-limited. Exposed in the Rust, Python, Go and JS SDKs +- guest-agent: `AttestGpu` runs NVIDIA GPU attestation on demand against a caller-supplied 32-byte nonce, so an application can re-check the device after a driver reload leaves a GPU that answers NVML but can no longer attest. Returns both the GPU-signed evidence (`nvattest collect-evidence`: the SPDM report and its certificate chain, which a third party can appraise with its own verifier) and the local verifier's appraisal of exactly those bytes (unsigned, `alg:none`, for callers inside the CVM). Neither binds the GPU to the TD -- an NVIDIA report binds the device and nonce and nothing else, so it stays relayable until TDISP/TEE-IO. Serialised and rate-limited. Exposed in the Rust, Python, Go and JS SDKs - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796) - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 28d41b459..ae0f80d57 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -4820,6 +4820,7 @@ dependencies = [ "anyhow", "serde", "serde_json", + "tempfile", "tokio", "tracing", "url", diff --git a/dstack/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto index 6c5fb472d..89d28f86c 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc.proto @@ -69,8 +69,9 @@ service DstackGuest { // the GPU -- a driver reload leaves a device that responds to NVML but can no // longer attest -- and before submitting work you care about. // - // It is NOT a remote attestation claim, and a relying party outside this CVM - // must not treat it as one. See `AttestGpuResponse.evidence`. + // Returns the GPU-signed evidence itself, not just a verdict about it, so a + // relying party can appraise it with its own verifier. That still does not + // bind the GPU to this TD -- see `AttestGpuResponse.evidence`. rpc AttestGpu(AttestGpuArgs) returns (AttestGpuResponse) {} // Sign a payload @@ -220,44 +221,41 @@ message AttestGpuArgs { } message AttestGpuResponse { - // Complete JSON output produced by nvattest, in the same shape `GpuInfo` - // returns, for the nonce in the request. + // GPU-signed evidence for the requested nonce: `nvattest collect-evidence` + // output, a JSON array with one entry per device carrying the base64 SPDM + // attestation report and its certificate chain. // - // dstack has already checked that nvattest succeeded and that every claim - // answers this nonce. Appraising the claims -- device count, CC state, VBIOS - // and driver versions, your own policy -- is the caller's job. + // This is the part a third party can check for itself, and the reason the RPC + // returns it rather than only a verdict. A relying party verifies the chain to + // NVIDIA's root, checks the report signature with the leaf key, confirms the + // nonce inside the report is the one it issued, and compares the measurements + // against NVIDIA's RIM documents -- with its own verifier, trusting nothing + // this CVM says. // - // To a caller inside this CVM this is a live, meaningful result: the agent ran - // NVIDIA's verifier, which checked the GPU's report signature, its certificate - // chain and OCSP status, and the driver and VBIOS RIM signatures, all against - // this nonce. - // - // It is NOT independently verifiable by a third party, for two separate - // reasons, and the first is the one that surprises people: - // - // 1. It is unsigned. `nvattest --verifier local` returns the verifier's - // conclusion, not the GPU's signed report. The detached EAT it embeds is - // `alg:none` with an empty signature, issued by `NVAT-LOCAL-VERIFIER`, and - // claims such as `x-nvidia-gpu-attestation-report-signature-verified` are - // assertions about a check already performed, not proof anyone can redo. - // The GPU's signed SPDM report and certificate chain are consumed during - // verification and do not appear here. So a relying party handed this JSON - // has nothing to check -- it is a self-report from inside the CVM. - // 2. Even signed, it would bind the device and the nonce but not the TD the - // device is attached to, so it can be relayed from a genuine remote GPU. - // Deriving the nonce from a TDX quote does not help; the relay can derive - // it too. Only TDISP/TEE-IO device binding closes that, and no current - // Hopper/Blackwell deployment offers it. - // - // Remote evidence therefore stays with the boot-time `gpu-attestation` runtime - // event. Its trust does not come from the JSON being self-authenticating - // either -- it comes from measured dstack code (pinned by `os_image_hash`) - // having done the appraisal before any workload existed, with sha256 of the - // exact bytes measured into RTMR3 and covered by the quote. + // What it still cannot establish is which TD the device is attached to. An + // NVIDIA report binds the device and the nonce and nothing else, so it can be + // relayed from a genuine remote GPU; deriving the nonce from a TDX quote does + // not help, because the relay can derive it too. Only TDISP/TEE-IO closes + // that, and no current Hopper/Blackwell deployment offers it. string evidence = 1; - // The nonce the GPU actually answered, hex-encoded, as it appears in - // `eat_nonce`. Echoed so a caller can assert on it without re-encoding. - string nonce = 2; + + // The local verifier's appraisal of exactly those bytes: `nvattest attest + // --verifier local` output, the same shape `GpuInfo` returns. + // + // Convenience for a caller inside this CVM, which is in the agent's trust + // domain: the agent checked the report signature, the certificate chain and + // OCSP status, and the driver and VBIOS RIM signatures, so the caller does not + // have to. It is NOT evidence and does not travel: the detached EAT it carries + // is `alg:none`, issued by `NVAT-LOCAL-VERIFIER`, and a claim such as + // `x-nvidia-gpu-attestation-report-signature-verified` is an assertion about a + // check already performed, not proof anyone else can redo. A remote party + // should ignore this field and appraise `evidence` itself. + string appraisal = 2; + + // The nonce both halves answer, hex-encoded, as it appears in `eat_nonce` and + // in each evidence entry. Echoed so a caller can assert on it without + // re-encoding. + string nonce = 3; } message GpuInfoResponse { diff --git a/dstack/guest-agent/src/gpu_attest.rs b/dstack/guest-agent/src/gpu_attest.rs index 2c28d8ee1..e3424cb9b 100644 --- a/dstack/guest-agent/src/gpu_attest.rs +++ b/dstack/guest-agent/src/gpu_attest.rs @@ -59,7 +59,7 @@ impl GpuAttestor { } /// Attest against `nonce`, or explain why not. - pub async fn attest(&self, nonce: &[u8]) -> Result { + pub async fn attest(&self, nonce: &[u8]) -> Result { if nonce.len() != nvattest::NONCE_LEN { bail!( "nonce must be exactly {} bytes, got {}", @@ -74,7 +74,8 @@ impl GpuAttestor { // starting a competing nvattest against the same devices. let mut last_run = self.last_run.lock().await; check_cooldown(*last_run, Instant::now(), self.cooldown)?; - let result = nvattest::attest(nonce, self.proxy_url.as_deref(), self.timeout).await; + let result = + nvattest::collect_and_appraise(nonce, self.proxy_url.as_deref(), self.timeout).await; // Stamp on failure too: a failing GPU is the case most likely to be // retried in a tight loop, and the most expensive to retry. *last_run = Some(Instant::now()); diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index f283fd1ef..db9e839fe 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -394,7 +394,8 @@ impl DstackGuestRpc for InternalRpcHandler { .await .context("GPU attestation failed")?; Ok(AttestGpuResponse { - evidence: String::from_utf8(attestation.output) + evidence: attestation.evidence, + appraisal: String::from_utf8(attestation.appraisal) .context("nvattest output is not valid UTF-8")?, nonce: attestation.nonce, }) diff --git a/dstack/nvattest/Cargo.toml b/dstack/nvattest/Cargo.toml index 672209a92..4694070af 100644 --- a/dstack/nvattest/Cargo.toml +++ b/dstack/nvattest/Cargo.toml @@ -19,6 +19,7 @@ serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["process", "time"] } tracing.workspace = true url.workspace = true +tempfile.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/nvattest/src/lib.rs b/dstack/nvattest/src/lib.rs index 00381b903..084b50b0b 100644 --- a/dstack/nvattest/src/lib.rs +++ b/dstack/nvattest/src/lib.rs @@ -53,6 +53,13 @@ struct NvattestOutput { claims: Vec, } +#[derive(Deserialize)] +struct CollectEvidenceOutput { + result_code: i64, + #[serde(default)] + evidences: Vec, +} + #[derive(Deserialize)] struct NonceClaim { #[serde(rename = "eat_nonce")] @@ -170,13 +177,7 @@ pub async fn run( if !available() { bail!("nvattest is not available in this image"); } - if nonce.len() != NONCE_LEN { - bail!( - "gpu attestation nonce must be {NONCE_LEN} bytes, got {}", - nonce.len() - ); - } - let nonce = hex_encode(nonce); + let nonce = check_nonce_len(nonce)?; let args = args(&nonce, proxy_url)?; if args.iter().any(|arg| arg == "--relying-party-policy") && !Path::new(TRUST_OUTPOST_POLICY).is_file() @@ -191,6 +192,118 @@ pub async fn run( Ok((nonce, output)) } +/// GPU-signed evidence plus the local verifier's appraisal of it. +/// +/// Kept apart because they are different things in the RATS sense: `evidence` +/// is what the GPU signed and anyone can check, `appraisal` is a verdict about +/// it that only the party who produced it vouches for. +#[derive(Debug)] +pub struct CollectedAttestation { + /// `collect-evidence` output: a JSON array with one entry per device, each + /// carrying the base64 SPDM attestation report and its certificate chain. + pub evidence: String, + /// `attest` output over exactly those bytes: the appraisal claims. + pub appraisal: Vec, + /// Hex nonce both halves answer. + pub nonce: String, + /// Parsed appraisal claims. + pub claims: Vec, +} + +/// Collect GPU-signed evidence for `nonce` without appraising it. +/// +/// Returns the `evidences` array alone, which is the shape `attest +/// --gpu-evidence-source=file` expects, and the shape a third party needs: the +/// report and certificate chain, not a verdict about them. +pub async fn collect_evidence(nonce: &[u8], timeout: Duration) -> Result<(String, Vec)> { + let nonce = check_nonce_len(nonce)?; + let args = [ + "collect-evidence", + "--device", + "gpu", + "--nonce", + &nonce, + "--format", + "json", + ]; + let output = run_command(NVATTEST, &args, timeout).await?; + if !output.stderr.is_empty() { + info!( + "nvattest collect-evidence: {}", + truncated_lossy(&output.stderr, 2048) + ); + } + check_status(&output)?; + let parsed: CollectEvidenceOutput = serde_json::from_slice(&output.stdout) + .context("failed to parse nvattest collect-evidence output")?; + if parsed.result_code != 0 { + bail!( + "nvattest collect-evidence failed (result_code={})", + parsed.result_code + ); + } + if parsed.evidences.is_empty() { + bail!("nvattest collected no GPU evidence"); + } + let evidences = + serde_json::to_vec(&parsed.evidences).context("failed to re-encode GPU evidence")?; + Ok((nonce, evidences)) +} + +/// Collect GPU-signed evidence and appraise those exact bytes. +/// +/// Two steps rather than one `attest` run so the caller gets evidence a third +/// party can check alongside the verdict, and so both provably describe the +/// same report: the appraisal reads back the collected bytes instead of asking +/// the GPU a second time. The SDK independently rejects an evidence file whose +/// nonce does not match the one passed here. +pub async fn collect_and_appraise( + nonce: &[u8], + proxy_url: Option<&str>, + timeout: Duration, +) -> Result { + if !available() { + bail!("nvattest is not available in this image"); + } + let (nonce, evidences) = collect_evidence(nonce, timeout).await?; + + let dir = tempfile::tempdir().context("failed to create a directory for GPU evidence")?; + let path = dir.path().join("gpu-evidence.json"); + std::fs::write(&path, &evidences).context("failed to stage GPU evidence")?; + let path = path.to_str().context("GPU evidence path is not UTF-8")?; + + let mut args = args(&nonce, proxy_url)?; + args.extend([ + "--gpu-evidence-source".to_string(), + "file".to_string(), + "--gpu-evidence-file".to_string(), + path.to_string(), + ]); + let borrowed = args.iter().map(String::as_str).collect::>(); + let output = run_command(NVATTEST, &borrowed, timeout).await?; + if !output.stderr.is_empty() { + info!("nvattest: {}", truncated_lossy(&output.stderr, 2048)); + } + check_status(&output)?; + let claims = check_nonce(&output.stdout, &nonce)?; + Ok(CollectedAttestation { + evidence: String::from_utf8(evidences).context("GPU evidence is not valid UTF-8")?, + appraisal: output.stdout, + nonce, + claims, + }) +} + +fn check_nonce_len(nonce: &[u8]) -> Result { + if nonce.len() != NONCE_LEN { + bail!( + "gpu attestation nonce must be {NONCE_LEN} bytes, got {}", + nonce.len() + ); + } + Ok(hex_encode(nonce)) +} + /// Turn a non-zero nvattest exit into an error carrying a bounded stderr tail. pub fn check_status(output: &Output) -> Result<()> { if !output.status.success() { @@ -293,6 +406,38 @@ mod tests { assert!(check_nonce(&output(0, &[]), &nonce).is_err()); } + #[tokio::test] + async fn collect_evidence_rejects_a_nonce_of_the_wrong_length() { + let err = collect_evidence(&[0u8; 16], DEFAULT_TIMEOUT) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("32 bytes") || err.contains("not available"), + "{err}" + ); + } + + #[test] + fn appraisal_args_read_back_the_collected_evidence() { + // The appraisal must consume the collected bytes rather than ask the + // GPU again, or the two halves of the response could describe different + // reports. Pin the flags that make that true. + let mut args = args("aa", None).unwrap(); + args.extend([ + "--gpu-evidence-source".to_string(), + "file".to_string(), + "--gpu-evidence-file".to_string(), + "/tmp/e.json".to_string(), + ]); + assert!(args + .windows(2) + .any(|w| w == ["--gpu-evidence-source", "file"])); + assert!(args + .windows(2) + .any(|w| w == ["--gpu-evidence-file", "/tmp/e.json"])); + } + #[tokio::test] async fn rejects_a_nonce_of_the_wrong_length() { let err = attest(&[0u8; 16], None, DEFAULT_TIMEOUT) diff --git a/sdk/curl/api.md b/sdk/curl/api.md index 3397afc3f..a2b53eea3 100644 --- a/sdk/curl/api.md +++ b/sdk/curl/api.md @@ -270,28 +270,32 @@ Use it after anything that may have reinitialised the GPU. A driver reload leave device that still answers NVML but can no longer attest, and this is how an application detects that before submitting work. -> [!WARNING] -> **This response cannot be independently verified by a third party.** Inside the CVM -> it is meaningful: the agent ran NVIDIA's verifier, which checked the GPU's report -> signature, certificate chain and OCSP status, and the driver and VBIOS RIM -> signatures, against your nonce. Outside it, two separate problems apply. +> [!IMPORTANT] +> `evidence` is checkable by anyone; `appraisal` is not. A relying party outside this +> CVM should verify `evidence` with its own verifier and ignore `appraisal`. +> +> **`evidence`** is `nvattest collect-evidence` output: one entry per device with the +> base64 SPDM attestation report and its certificate chain, signed by the GPU over the +> nonce you sent. To check it: verify the chain to NVIDIA's device-identity root, +> verify the report signature with the leaf key, confirm the nonce inside the report is +> the one you issued, then compare the measurements against NVIDIA's RIM documents. > -> 1. **It is unsigned.** `--verifier local` returns the verifier's *conclusion*, not -> the GPU's signed report. The embedded detached EAT is `alg:none` with an empty -> signature, issued by `NVAT-LOCAL-VERIFIER`, and claims like -> `x-nvidia-gpu-attestation-report-signature-verified: true` are assertions about a -> check already performed — the signed SPDM report and certificate chain are -> consumed during verification and are not carried in the output. A relying party -> handed this JSON has nothing to check. -> 2. **No TD binding.** Even signed, an NVIDIA report binds the device and the nonce, -> not the TD it is attached to, so it can be relayed from a genuine remote GPU. -> Deriving the nonce from a TDX quote does not help — the relay can derive it too. -> Only TDISP/TEE-IO closes this, and no current Hopper/Blackwell deployment has it. +> **`appraisal`** is the local verifier's verdict on exactly those bytes, provided +> because a caller inside the CVM is in the agent's trust domain and usually just wants +> the answer. It does not travel: its detached EAT is `alg:none` issued by +> `NVAT-LOCAL-VERIFIER`, and a claim like +> `x-nvidia-gpu-attestation-report-signature-verified: true` is an assertion about a +> check already performed, not proof anyone can redo. + +> [!WARNING] +> Neither field binds the GPU to *this* CVM. An NVIDIA report binds the device and the +> nonce and nothing else, so it can be relayed from a genuine remote GPU; deriving the +> nonce from a TDX quote does not help, because the relay can derive it too. Only +> TDISP/TEE-IO closes this, and no current Hopper/Blackwell deployment offers it. > -> Use it as a local health check. For remote evidence use the boot-time -> `gpu-attestation` runtime event: its trust comes not from the JSON being -> self-authenticating but from measured dstack code (pinned by `os_image_hash`) having -> appraised the GPU before any workload existed, with the digest in RTMR3. +> For evidence that the GPU is bound to this TD, use the boot-time `gpu-attestation` +> runtime event: measured dstack code (pinned by `os_image_hash`) appraised the GPU +> before any workload existed, with the digest in RTMR3 under the quote. **Endpoint:** `/AttestGpu` @@ -314,13 +318,15 @@ curl --unix-socket /var/run/dstack.sock -X POST \ **Response:** ```json { - "evidence": "{\"result_code\": 0, \"claims\": [...]}", + "evidence": "[{\"arch\": \"HOPPER\", \"nonce\": \"abab...\", \"evidence\": \"\", \"certificate\": \"\"}]", + "appraisal": "{\"result_code\": 0, \"claims\": [...]}", "nonce": "abababababababababababababababababababababababababababababababab" } ``` -dstack has already checked that nvattest succeeded and that every claim answers your -nonce; appraising the claims is yours. Calls are serialised and rate-limited (one +The two halves describe the same report: dstack collects the evidence once and appraises +those exact bytes rather than asking the GPU twice. dstack has already checked that +nvattest succeeded and that every claim answers your nonce. Calls are serialised and rate-limited (one attestation per 10s), because each one spawns `nvattest` and fetches OCSP and RIM collateral from NVIDIA. A call arriving inside the cooldown is rejected with a wait hint rather than queued. diff --git a/sdk/go/README.md b/sdk/go/README.md index b9738bef9..8d2606514 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -589,14 +589,15 @@ if err != nil { fmt.Println(resp.Evidence) ``` -> [!WARNING] -> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — -> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: -> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with -> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and -> even signed it would bind the device and nonce but not the TD, so it is relayable. -> Use it as a local health check; for remote evidence use the boot-time -> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. +> [!IMPORTANT] +> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its +> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on +> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should +> appraise `evidence` itself and ignore `appraisal`. +> +> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and +> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the +> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. ##### `GpuInfo(ctx context.Context) (*GpuInfoResponse, error)` diff --git a/sdk/go/dstack/client.go b/sdk/go/dstack/client.go index 50b5f6a7f..724f99781 100644 --- a/sdk/go/dstack/client.go +++ b/sdk/go/dstack/client.go @@ -134,9 +134,12 @@ type GpuInfoResponse struct { // evidence use the boot-time `gpu-attestation` event, which measured code emits // before any workload exists and which the event log binds to the quote. type AttestGpuResponse struct { - // Evidence is the complete nvattest JSON for the requested nonce. + // Evidence is GPU-signed: collect-evidence output, one entry per device + // carrying the base64 SPDM attestation report and its certificate chain. Evidence string `json:"evidence"` - // Nonce is the nonce the GPU answered, hex-encoded, as it appears in eat_nonce. + // Appraisal is the local verifier's verdict on exactly those bytes. Unsigned. + Appraisal string `json:"appraisal"` + // Nonce is the nonce both halves answer, hex-encoded, as it appears in eat_nonce. Nonce string `json:"nonce"` } diff --git a/sdk/go/dstack/client_test.go b/sdk/go/dstack/client_test.go index 7bbd91f78..380dfc073 100644 --- a/sdk/go/dstack/client_test.go +++ b/sdk/go/dstack/client_test.go @@ -80,7 +80,7 @@ func TestAttest(t *testing.T) { } func TestAttestGpu(t *testing.T) { - const evidence = `{"result_code":0,"claims":[]}` + const evidence = `[{"arch":"HOPPER","nonce":"ab","evidence":"BASE64","certificate":"BASE64"}]` nonce := bytes.Repeat([]byte{0xab}, 32) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/AttestGpu" { @@ -95,8 +95,9 @@ func TestAttestGpu(t *testing.T) { } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{ - "evidence": evidence, - "nonce": hex.EncodeToString(nonce), + "evidence": evidence, + "appraisal": `{"result_code":0,"claims":[]}`, + "nonce": hex.EncodeToString(nonce), }) })) defer server.Close() @@ -109,6 +110,9 @@ func TestAttestGpu(t *testing.T) { if resp.Evidence != evidence { t.Fatalf("unexpected evidence: %s", resp.Evidence) } + if resp.Appraisal == "" { + t.Fatal("expected the appraisal to be carried alongside the evidence") + } if resp.Nonce != hex.EncodeToString(nonce) { t.Fatalf("unexpected nonce: %s", resp.Nonce) } diff --git a/sdk/js/README.md b/sdk/js/README.md index 36162cbd9..a0063a335 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -105,14 +105,15 @@ answers NVML but can no longer attest. const { evidence } = await client.attestGpu(crypto.randomBytes(32)) ``` -> [!WARNING] -> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — -> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: -> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with -> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and -> even signed it would bind the device and nonce but not the TD, so it is relayable. -> Use it as a local health check; for remote evidence use the boot-time -> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. +> [!IMPORTANT] +> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its +> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on +> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should +> appraise `evidence` itself and ignore `appraisal`. +> +> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and +> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the +> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. ### `gpuInfo()` diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index 93e53e22f..0f1924a22 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -106,27 +106,33 @@ export interface AttestResponse { /** * Result of a fresh, on-demand NVIDIA GPU attestation. * - * Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked - * the GPU's report signature, its certificate chain and OCSP status, and the driver and - * VBIOS RIM signatures, all against this nonce. + * `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report + * and its certificate chain, per device, for the nonce you sent. A relying party + * verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce + * inside the report, and compares measurements against NVIDIA's RIM documents, using + * its own verifier and trusting nothing the CVM says. * - * NOT independently verifiable by a third party, for two separate reasons. First, it is - * unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed - * report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim - * like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; - * the signed artifacts are consumed during verification and not carried here. Second, - * even signed it would bind the device and the nonce but not the TD, so it is relayable - * from a genuine remote GPU. Use it as a local health check. For remote evidence use the - * boot-time `gpu-attestation` event, which measured code emits before any workload - * exists and which the event log binds to the quote. + * `appraisal` is the local verifier's verdict on those same bytes -- convenient inside + * the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, + * so a remote party should ignore it and appraise `evidence` itself. + * + * Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and + * nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would + * close that. */ export interface AttestGpuResponse { __name__: Readonly<'AttestGpuResponse'> - /** Complete nvattest JSON for the requested nonce. */ + /** + * GPU-signed evidence: `collect-evidence` output, one entry per device carrying + * the base64 SPDM attestation report and its certificate chain. + */ evidence: string - /** The nonce the GPU answered, hex-encoded, as it appears in `eat_nonce`. */ + /** The local verifier's verdict on exactly those bytes. Unsigned. */ + appraisal: string + + /** The nonce both halves answer, hex-encoded, as it appears in `eat_nonce`. */ nonce: string } @@ -339,7 +345,7 @@ export class DstackClient { throw new Error(`Nonce must be exactly 32 bytes, got ${nonce.length}.`) } const payload = JSON.stringify({ nonce: to_hex(nonce) }) - const result = await send_rpc_request<{ evidence: string, nonce: string }>(this.endpoint, '/AttestGpu', payload) + const result = await send_rpc_request<{ evidence: string, appraisal: string, nonce: string }>(this.endpoint, '/AttestGpu', payload) if ('error' in (result as any)) { throw new Error((result as any)['error'] as string) } diff --git a/sdk/python/README.md b/sdk/python/README.md index 4d6c50909..f7cb45df8 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -111,14 +111,15 @@ result = client.attest_gpu(os.urandom(32)) print(result.evidence) ``` -> [!WARNING] -> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — -> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: -> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with -> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and -> even signed it would bind the device and nonce but not the TD, so it is relayable. -> Use it as a local health check; for remote evidence use the boot-time -> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. +> [!IMPORTANT] +> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its +> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on +> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should +> appraise `evidence` itself and ignore `appraisal`. +> +> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and +> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the +> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. ### GPU Info diff --git a/sdk/python/src/dstack_sdk/dstack_client.py b/sdk/python/src/dstack_sdk/dstack_client.py index 5f60d5c44..9ee27f8a8 100644 --- a/sdk/python/src/dstack_sdk/dstack_client.py +++ b/sdk/python/src/dstack_sdk/dstack_client.py @@ -160,23 +160,24 @@ def decode_attestation(self) -> bytes: class AttestGpuResponse(BaseModel): """Result of a fresh, on-demand NVIDIA GPU attestation. - Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked - the GPU's report signature, its certificate chain and OCSP status, and the driver and - VBIOS RIM signatures, all against this nonce. - - NOT independently verifiable by a third party, for two separate reasons. First, it is - unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed - report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim - like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; - the signed artifacts are consumed during verification and not carried here. Second, - even signed it would bind the device and the nonce but not the TD, so it is relayable - from a genuine remote GPU. Use it as a local health check. For remote evidence use the - boot-time `gpu-attestation` event, which measured code emits before any workload - exists and which the event log binds to the quote. + `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report + and its certificate chain, per device, for the nonce you sent. A relying party + verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce + inside the report, and compares measurements against NVIDIA's RIM documents, using + its own verifier and trusting nothing the CVM says. + + `appraisal` is the local verifier's verdict on those same bytes -- convenient inside + the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, + so a remote party should ignore it and appraise `evidence` itself. + + Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and + nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would + close that. """ evidence: str nonce: str + appraisal: str = "" class GpuInfoResponse(BaseModel): @@ -465,19 +466,19 @@ async def attest( async def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. - Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked - the GPU's report signature, its certificate chain and OCSP status, and the driver and - VBIOS RIM signatures, all against this nonce. - - NOT independently verifiable by a third party, for two separate reasons. First, it is - unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed - report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim - like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; - the signed artifacts are consumed during verification and not carried here. Second, - even signed it would bind the device and the nonce but not the TD, so it is relayable - from a genuine remote GPU. Use it as a local health check. For remote evidence use the - boot-time `gpu-attestation` event, which measured code emits before any workload - exists and which the event log binds to the quote. + `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report + and its certificate chain, per device, for the nonce you sent. A relying party + verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce + inside the report, and compares measurements against NVIDIA's RIM documents, using + its own verifier and trusting nothing the CVM says. + + `appraisal` is the local verifier's verdict on those same bytes -- convenient inside + the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, + so a remote party should ignore it and appraise `evidence` itself. + + Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and + nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would + close that. """ if not isinstance(nonce, (bytes, bytearray)) or len(nonce) != 32: raise ValueError("nonce must be exactly 32 bytes") @@ -623,19 +624,19 @@ def attest( def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. - Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked - the GPU's report signature, its certificate chain and OCSP status, and the driver and - VBIOS RIM signatures, all against this nonce. - - NOT independently verifiable by a third party, for two separate reasons. First, it is - unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed - report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim - like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; - the signed artifacts are consumed during verification and not carried here. Second, - even signed it would bind the device and the nonce but not the TD, so it is relayable - from a genuine remote GPU. Use it as a local health check. For remote evidence use the - boot-time `gpu-attestation` event, which measured code emits before any workload - exists and which the event log binds to the quote. + `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report + and its certificate chain, per device, for the nonce you sent. A relying party + verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce + inside the report, and compares measurements against NVIDIA's RIM documents, using + its own verifier and trusting nothing the CVM says. + + `appraisal` is the local verifier's verdict on those same bytes -- convenient inside + the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, + so a remote party should ignore it and appraise `evidence` itself. + + Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and + nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would + close that. """ raise NotImplementedError diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 40a937df8..ef6534c37 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -124,19 +124,21 @@ async def test_async_client_attest(): @pytest.mark.asyncio async def test_async_client_attest_gpu(monkeypatch): - evidence = '{"result_code":0,"claims":[]}' + evidence = '[{"arch":"HOPPER","evidence":"BASE64","certificate":"BASE64"}]' + appraisal = '{"result_code":0,"claims":[]}' nonce = bytes([0xAB]) * 32 async def fake_send(self, method, payload): assert method == "AttestGpu" assert payload == {"nonce": nonce.hex()} - return {"evidence": evidence, "nonce": nonce.hex()} + return {"evidence": evidence, "appraisal": appraisal, "nonce": nonce.hex()} monkeypatch.setenv("DSTACK_SIMULATOR_ENDPOINT", "http://localhost:0") monkeypatch.setattr(AsyncDstackClient, "_send_rpc_request", fake_send) result = await AsyncDstackClient().attest_gpu(nonce) assert isinstance(result, AttestGpuResponse) assert result.evidence == evidence + assert result.appraisal == appraisal assert result.nonce == nonce.hex() diff --git a/sdk/rust/README.md b/sdk/rust/README.md index 9ba0fdf6d..fbe0a8f12 100644 --- a/sdk/rust/README.md +++ b/sdk/rust/README.md @@ -115,14 +115,15 @@ let result = client.attest_gpu(nonce.to_vec()).await?; println!("{}", result.evidence); ``` -> [!WARNING] -> **Not independently verifiable by a third party.** Inside the CVM it is meaningful — -> the agent ran NVIDIA's verifier against your nonce. Outside it, two things are true: -> the output is *unsigned* (`--verifier local` returns the verifier's conclusion, with -> an `alg:none` detached EAT; the GPU's signed report is consumed and not carried), and -> even signed it would bind the device and nonce but not the TD, so it is relayable. -> Use it as a local health check; for remote evidence use the boot-time -> `gpu-attestation` event bound to the quote. Calls are rate-limited to one per 10s. +> [!IMPORTANT] +> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its +> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on +> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should +> appraise `evidence` itself and ignore `appraisal`. +> +> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and +> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the +> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. #### `gpu_info() -> GpuInfoResponse` diff --git a/sdk/rust/src/dstack_client.rs b/sdk/rust/src/dstack_client.rs index ac13ccb15..31e11e7d3 100644 --- a/sdk/rust/src/dstack_client.rs +++ b/sdk/rust/src/dstack_client.rs @@ -166,19 +166,19 @@ impl DstackClient { /// Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. /// - /// Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked - /// the GPU's report signature, its certificate chain and OCSP status, and the driver and - /// VBIOS RIM signatures, all against this nonce. + /// `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report + /// and its certificate chain, per device, for the nonce you sent. A relying party + /// verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce + /// inside the report, and compares measurements against NVIDIA's RIM documents, using + /// its own verifier and trusting nothing the CVM says. /// - /// NOT independently verifiable by a third party, for two separate reasons. First, it is - /// unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed - /// report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim - /// like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; - /// the signed artifacts are consumed during verification and not carried here. Second, - /// even signed it would bind the device and the nonce but not the TD, so it is relayable - /// from a genuine remote GPU. Use it as a local health check. For remote evidence use the - /// boot-time `gpu-attestation` event, which measured code emits before any workload - /// exists and which the event log binds to the quote. + /// `appraisal` is the local verifier's verdict on those same bytes -- convenient inside + /// the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, + /// so a remote party should ignore it and appraise `evidence` itself. + /// + /// Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and + /// nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would + /// close that. pub async fn attest_gpu(&self, nonce: Vec) -> Result { if nonce.len() != 32 { anyhow::bail!("Nonce must be exactly 32 bytes") diff --git a/sdk/rust/types/src/dstack.rs b/sdk/rust/types/src/dstack.rs index 2d569c6e2..13804c124 100644 --- a/sdk/rust/types/src/dstack.rs +++ b/sdk/rust/types/src/dstack.rs @@ -100,26 +100,30 @@ pub struct AttestResponse { /// Response from a fresh, on-demand NVIDIA GPU attestation. /// -/// Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which checked -/// the GPU's report signature, its certificate chain and OCSP status, and the driver and -/// VBIOS RIM signatures, all against this nonce. +/// `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report +/// and its certificate chain, per device, for the nonce you sent. A relying party +/// verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce +/// inside the report, and compares measurements against NVIDIA's RIM documents, using +/// its own verifier and trusting nothing the CVM says. /// -/// NOT independently verifiable by a third party, for two separate reasons. First, it is -/// unsigned: `--verifier local` returns the verifier's conclusion, not the GPU's signed -/// report. Its detached EAT is `alg:none` issued by `NVAT-LOCAL-VERIFIER`, and a claim -/// like `x-nvidia-gpu-attestation-report-signature-verified` is an assertion, not proof; -/// the signed artifacts are consumed during verification and not carried here. Second, -/// even signed it would bind the device and the nonce but not the TD, so it is relayable -/// from a genuine remote GPU. Use it as a local health check. For remote evidence use the -/// boot-time `gpu-attestation` event, which measured code emits before any workload -/// exists and which the event log binds to the quote. +/// `appraisal` is the local verifier's verdict on those same bytes -- convenient inside +/// the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, +/// so a remote party should ignore it and appraise `evidence` itself. +/// +/// Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and +/// nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would +/// close that. #[derive(Debug, Serialize, Deserialize)] #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] #[cfg_attr(feature = "borsh_schema", derive(BorshSchema))] pub struct AttestGpuResponse { - /// Complete JSON output produced by nvattest for the requested nonce. + /// GPU-signed evidence: `collect-evidence` output, one entry per device + /// carrying the base64 SPDM attestation report and its certificate chain. pub evidence: String, - /// The nonce the GPU answered, hex-encoded, as it appears in `eat_nonce`. + /// The local verifier's appraisal of exactly those bytes. Unsigned. + #[serde(default)] + pub appraisal: String, + /// The nonce both halves answer, hex-encoded, as it appears in `eat_nonce`. pub nonce: String, } From 96a0b0e1b0b8920c495768c6572a3015d113eb95 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 23 Aug 2026 23:47:16 -0700 Subject: [PATCH 5/7] refactor(gpu): return vendor-neutral evidence bundles --- CHANGELOG.md | 2 +- dstack/Cargo.lock | 1 - dstack/guest-agent/rpc/proto/agent_rpc.proto | 58 ++++-------- dstack/guest-agent/src/gpu_attest.rs | 93 +++----------------- dstack/guest-agent/src/rpc_service.rs | 22 +++-- dstack/nvattest/Cargo.toml | 1 - dstack/nvattest/src/lib.rs | 82 ----------------- sdk/curl/api.md | 66 +++----------- sdk/go/README.md | 27 +++--- sdk/go/dstack/client.go | 29 ++---- sdk/go/dstack/client_test.go | 18 ++-- sdk/js/README.md | 22 ++--- sdk/js/src/index.ts | 36 ++------ sdk/python/README.md | 21 ++--- sdk/python/src/dstack_sdk/__init__.py | 2 + sdk/python/src/dstack_sdk/dstack_client.py | 60 +++---------- sdk/python/tests/test_client.py | 13 +-- sdk/rust/README.md | 21 ++--- sdk/rust/src/dstack_client.rs | 17 +--- sdk/rust/types/src/dstack.rs | 34 +++---- 20 files changed, 152 insertions(+), 473 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f4121142..b42a5dcd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - sdk: `verify_signature` and `verify_signature_chain` in all four SDKs, replacing the removed guest-agent `Verify` RPC. `verify_signature_chain` is new capability rather than a port: it walks all three links of a `Sign` signature chain -- payload signature, the app root key attesting `"{purpose}:{hex(pubkey)}"`, and the KMS root attesting that app root for this `app_id` -- and requires the chain to anchor at a KMS root public key **the caller supplies**. That anchor has to come from somewhere independently trusted (the `DstackKms` contract's `kmsInfo().k256Pubkey`, or a pinned value); read it from the KMS being checked and an attacker who can answer that query can also mint a self-consistent chain. The four ports are pinned against one committed set of test vectors, `sdk/tests/vectors/signature_chain.json`, generated from the real KMS and guest-agent primitives -- this repo has shipped cross-language crypto drift twice already -- guest-agent: `AttestGpu` runs NVIDIA GPU attestation on demand against a caller-supplied 32-byte nonce, so an application can re-check the device after a driver reload leaves a GPU that answers NVML but can no longer attest. Returns both the GPU-signed evidence (`nvattest collect-evidence`: the SPDM report and its certificate chain, which a third party can appraise with its own verifier) and the local verifier's appraisal of exactly those bytes (unsigned, `alg:none`, for callers inside the CVM). Neither binds the GPU to the TD -- an NVIDIA report binds the device and nonce and nothing else, so it stays relayable until TDISP/TEE-IO. Serialised and rate-limited. Exposed in the Rust, Python, Go and JS SDKs +- guest-agent: `AttestGpu` collects vendor-native GPU evidence on demand against a caller-supplied 32-byte nonce. It returns opaque, versioned evidence bundles identified by vendor and format for independent appraisal. The response format is extensible to additional GPU vendors. Exposed in the Rust, Python, Go, and JS SDKs - sdk: `AppCompose` in the Go SDK gained `init_script`, `storage_fs`, `swap_size`, `event_log_version`, `port_policy` and `verity_volumes`, and `Requirements` gained `gpu_policy` in the Go and Python SDKs - shared API authentication (`dstack-api-auth`) protecting the full VMM HTTP/pRPC/UI surface and unifying Gateway/KMS admin auth: bearer/`X-Admin-Token`/HTTP Basic/bcrypt htpasswd, constant-time verification (#796) - gateway: `Admin.Status` reports `health_gating`, so an operator can see whether this node's health polling is switched on. With it off, instances that opted in sit at `unknown` forever and are all in rotation, which is otherwise indistinguishable on the dashboard from being held out pending a first answer diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index ae0f80d57..28d41b459 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -4820,7 +4820,6 @@ dependencies = [ "anyhow", "serde", "serde_json", - "tempfile", "tokio", "tracing", "url", diff --git a/dstack/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto index 89d28f86c..9d3ef3d78 100644 --- a/dstack/guest-agent/rpc/proto/agent_rpc.proto +++ b/dstack/guest-agent/rpc/proto/agent_rpc.proto @@ -61,7 +61,7 @@ service DstackGuest { // Get GPU information collected during boot. rpc GpuInfo(google.protobuf.Empty) returns (GpuInfoResponse) {} - // Run NVIDIA GPU attestation now, against a nonce the caller chooses. + // Collect GPU attestation evidence now, against a nonce the caller chooses. // // This answers "is the device I can talk to right now a genuine, CC-enabled // NVIDIA GPU that signs my challenge", which `GpuInfo` cannot: that returns a @@ -69,9 +69,9 @@ service DstackGuest { // the GPU -- a driver reload leaves a device that responds to NVML but can no // longer attest -- and before submitting work you care about. // - // Returns the GPU-signed evidence itself, not just a verdict about it, so a - // relying party can appraise it with its own verifier. That still does not - // bind the GPU to this TD -- see `AttestGpuResponse.evidence`. + // Returns vendor-native evidence, not a local verdict, so a relying party can + // appraise it with its own verifier. Evidence still does not bind the GPU to + // this TD; see `AttestGpuResponse.bundles`. rpc AttestGpu(AttestGpuArgs) returns (AttestGpuResponse) {} // Sign a payload @@ -221,41 +221,21 @@ message AttestGpuArgs { } message AttestGpuResponse { - // GPU-signed evidence for the requested nonce: `nvattest collect-evidence` - // output, a JSON array with one entry per device carrying the base64 SPDM - // attestation report and its certificate chain. - // - // This is the part a third party can check for itself, and the reason the RPC - // returns it rather than only a verdict. A relying party verifies the chain to - // NVIDIA's root, checks the report signature with the leaf key, confirms the - // nonce inside the report is the one it issued, and compares the measurements - // against NVIDIA's RIM documents -- with its own verifier, trusting nothing - // this CVM says. - // - // What it still cannot establish is which TD the device is attached to. An - // NVIDIA report binds the device and the nonce and nothing else, so it can be - // relayed from a genuine remote GPU; deriving the nonce from a TDX quote does - // not help, because the relay can derive it too. Only TDISP/TEE-IO closes - // that, and no current Hopper/Blackwell deployment offers it. - string evidence = 1; - - // The local verifier's appraisal of exactly those bytes: `nvattest attest - // --verifier local` output, the same shape `GpuInfo` returns. - // - // Convenience for a caller inside this CVM, which is in the agent's trust - // domain: the agent checked the report signature, the certificate chain and - // OCSP status, and the driver and VBIOS RIM signatures, so the caller does not - // have to. It is NOT evidence and does not travel: the detached EAT it carries - // is `alg:none`, issued by `NVAT-LOCAL-VERIFIER`, and a claim such as - // `x-nvidia-gpu-attestation-report-signature-verified` is an assertion about a - // check already performed, not proof anyone else can redo. A remote party - // should ignore this field and appraise `evidence` itself. - string appraisal = 2; - - // The nonce both halves answer, hex-encoded, as it appears in `eat_nonce` and - // in each evidence entry. Echoed so a caller can assert on it without - // re-encoding. - string nonce = 3; + // Vendor-native evidence bundles. The caller must select a verifier using + // `vendor` and `format`, then verify the signature, certificate chain, + // measurements, and the nonce embedded in the evidence. + repeated GpuEvidenceBundle bundles = 1; +} + +message GpuEvidenceBundle { + // Stable GPU vendor identifier, for example `nvidia`, `amd`, or `intel`. + string vendor = 1; + + // Vendor-specific evidence format and version. + string format = 2; + + // Opaque vendor-native evidence bytes. Do not assume UTF-8 or JSON. + bytes evidence = 3; } message GpuInfoResponse { diff --git a/dstack/guest-agent/src/gpu_attest.rs b/dstack/guest-agent/src/gpu_attest.rs index e3424cb9b..ced4dcd6f 100644 --- a/dstack/guest-agent/src/gpu_attest.rs +++ b/dstack/guest-agent/src/gpu_attest.rs @@ -5,61 +5,34 @@ //! On-demand GPU attestation for the `AttestGpu` RPC. //! //! Running `nvattest` is expensive in a way most RPCs are not: it spawns a -//! process, fetches OCSP and RIM collateral from NVIDIA (the SDK's cache is -//! per-process, so a subprocess starts cold every time), and can take minutes -//! on a slow link. The agent's socket is mode 0777, so any container in the -//! CVM can reach it. -//! -//! The CVM is a single trust domain, so a container hammering this endpoint is -//! only hurting its own deployment -- that is not the concern. The concern is -//! that a retry loop would hammer *NVIDIA's* services from every dstack CVM -//! running the buggy app, and would keep a wedged GPU tool respawning. So the -//! gate below serialises attestations and enforces a cooldown between them. +//! process and talks to the GPU through the driver. The gate below serialises +//! collection so concurrent callers do not compete for the same devices. -use std::time::{Duration, Instant}; +use std::time::Duration; use anyhow::{bail, Result}; use tokio::sync::Mutex; -/// Minimum spacing between two attestations. Chosen against what the call -/// actually costs: a cold collateral fetch is seconds, so a caller polling -/// faster than this is not learning anything new, only generating load. -pub const COOLDOWN: Duration = Duration::from_secs(10); - -/// Serialises `nvattest` runs and rate-limits them. +/// Serialises GPU evidence collection. /// /// Results are deliberately not shared between waiters: each caller supplies /// its own nonce, and handing back evidence that answers somebody else's /// challenge is exactly the confusion this API exists to avoid. pub struct GpuAttestor { - proxy_url: Option, timeout: Duration, - cooldown: Duration, - last_run: Mutex>, + run_lock: Mutex<()>, } impl GpuAttestor { - pub fn new(proxy_url: Option) -> Self { - Self { - proxy_url, - timeout: nvattest::DEFAULT_TIMEOUT, - cooldown: COOLDOWN, - last_run: Mutex::new(None), - } - } - - #[cfg(test)] - fn with_cooldown(cooldown: Duration) -> Self { + pub fn new() -> Self { Self { - proxy_url: None, timeout: nvattest::DEFAULT_TIMEOUT, - cooldown, - last_run: Mutex::new(None), + run_lock: Mutex::new(()), } } /// Attest against `nonce`, or explain why not. - pub async fn attest(&self, nonce: &[u8]) -> Result { + pub async fn attest(&self, nonce: &[u8]) -> Result> { if nonce.len() != nvattest::NONCE_LEN { bail!( "nonce must be exactly {} bytes, got {}", @@ -72,62 +45,20 @@ impl GpuAttestor { } // Held across the whole run, so a second caller waits rather than // starting a competing nvattest against the same devices. - let mut last_run = self.last_run.lock().await; - check_cooldown(*last_run, Instant::now(), self.cooldown)?; - let result = - nvattest::collect_and_appraise(nonce, self.proxy_url.as_deref(), self.timeout).await; - // Stamp on failure too: a failing GPU is the case most likely to be - // retried in a tight loop, and the most expensive to retry. - *last_run = Some(Instant::now()); - result - } -} - -/// Reject a call that arrives before the cooldown has elapsed. -fn check_cooldown(last_run: Option, now: Instant, cooldown: Duration) -> Result<()> { - let Some(last_run) = last_run else { - return Ok(()); - }; - let elapsed = now.saturating_duration_since(last_run); - if elapsed < cooldown { - let wait = (cooldown - elapsed).as_secs() + 1; - bail!("GPU attestation was run too recently; retry in {wait}s"); + let _guard = self.run_lock.lock().await; + let (_, evidence) = nvattest::collect_evidence(nonce, self.timeout).await?; + Ok(evidence) } - Ok(()) } #[cfg(test)] mod tests { use super::*; - #[test] - fn first_call_is_always_allowed() { - assert!(check_cooldown(None, Instant::now(), COOLDOWN).is_ok()); - } - - #[test] - fn a_call_inside_the_cooldown_is_rejected_with_a_wait_hint() { - let now = Instant::now(); - let last = now - Duration::from_secs(3); - let err = check_cooldown(Some(last), now, Duration::from_secs(10)) - .unwrap_err() - .to_string(); - assert!(err.contains("retry in"), "{err}"); - } - - #[test] - fn a_call_after_the_cooldown_is_allowed() { - let now = Instant::now(); - let last = now - Duration::from_secs(11); - assert!(check_cooldown(Some(last), now, Duration::from_secs(10)).is_ok()); - } - #[tokio::test] async fn a_wrong_length_nonce_is_rejected_before_anything_expensive() { - let attestor = GpuAttestor::with_cooldown(Duration::from_secs(0)); + let attestor = GpuAttestor::new(); let err = attestor.attest(&[0u8; 16]).await.unwrap_err().to_string(); assert!(err.contains("exactly 32 bytes"), "{err}"); - // Rejected on arity, so it must not have consumed the rate limit. - assert!(attestor.last_run.lock().await.is_none()); } } diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index db9e839fe..41bf3987f 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -17,8 +17,8 @@ use dstack_guest_agent_rpc::{ worker_server::{WorkerRpc, WorkerServer}, AppInfo, AttestAppKeyRequest, AttestGpuArgs, AttestGpuResponse, AttestResponse, DeriveK256KeyResponse, DeriveKeyArgs, GetKeyArgs, GetKeyResponse, GetQuoteResponse, - GetTlsKeyArgs, GetTlsKeyResponse, GpuInfoResponse, HealthResponse, RawQuoteArgs, SignRequest, - SignResponse, TdxQuoteArgs, TdxQuoteResponse, WorkerVersion, + GetTlsKeyArgs, GetTlsKeyResponse, GpuEvidenceBundle, GpuInfoResponse, HealthResponse, + RawQuoteArgs, SignRequest, SignResponse, TdxQuoteArgs, TdxQuoteResponse, WorkerVersion, }; use dstack_types::{AppKeys, SysConfig, GPU_ATTESTATION_OUTPUT}; use ed25519_dalek::ed25519::signature::hazmat::PrehashSigner; @@ -164,10 +164,7 @@ impl AppState { serde_json::from_str(&fs::read_to_string(&config.sys_config_file)?) .context("Failed to parse VM config")?; let collateral_urls = sys_config.collateral_urls(); - // Same collateral proxy the boot gate uses, so a CVM without egress to - // NVIDIA can still attest on demand. - let gpu_attestor = - crate::gpu_attest::GpuAttestor::new(sys_config.nvidia_attestation_proxy_url.clone()); + let gpu_attestor = crate::gpu_attest::GpuAttestor::new(); let vm_config = sys_config.vm_config; // Same trust anchor decision as dstack-util: never host-supplied, and // development roots only when this guest published them itself. @@ -386,7 +383,7 @@ impl DstackGuestRpc for InternalRpcHandler { } async fn attest_gpu(self, request: AttestGpuArgs) -> Result { - let attestation = self + let evidence = self .state .inner .gpu_attestor @@ -394,10 +391,11 @@ impl DstackGuestRpc for InternalRpcHandler { .await .context("GPU attestation failed")?; Ok(AttestGpuResponse { - evidence: attestation.evidence, - appraisal: String::from_utf8(attestation.appraisal) - .context("nvattest output is not valid UTF-8")?, - nonce: attestation.nonce, + bundles: vec![GpuEvidenceBundle { + vendor: "nvidia".to_string(), + format: "nvidia-nvattest-collect-evidence-json-v1".to_string(), + evidence, + }], }) } @@ -1005,7 +1003,7 @@ pNs85uhOZE8z2jr8Pg== }, }), health: None, - gpu_attestor: crate::gpu_attest::GpuAttestor::new(None), + gpu_attestor: crate::gpu_attest::GpuAttestor::new(), }; ( diff --git a/dstack/nvattest/Cargo.toml b/dstack/nvattest/Cargo.toml index 4694070af..672209a92 100644 --- a/dstack/nvattest/Cargo.toml +++ b/dstack/nvattest/Cargo.toml @@ -19,7 +19,6 @@ serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["process", "time"] } tracing.workspace = true url.workspace = true -tempfile.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/nvattest/src/lib.rs b/dstack/nvattest/src/lib.rs index 084b50b0b..f933ecd66 100644 --- a/dstack/nvattest/src/lib.rs +++ b/dstack/nvattest/src/lib.rs @@ -192,24 +192,6 @@ pub async fn run( Ok((nonce, output)) } -/// GPU-signed evidence plus the local verifier's appraisal of it. -/// -/// Kept apart because they are different things in the RATS sense: `evidence` -/// is what the GPU signed and anyone can check, `appraisal` is a verdict about -/// it that only the party who produced it vouches for. -#[derive(Debug)] -pub struct CollectedAttestation { - /// `collect-evidence` output: a JSON array with one entry per device, each - /// carrying the base64 SPDM attestation report and its certificate chain. - pub evidence: String, - /// `attest` output over exactly those bytes: the appraisal claims. - pub appraisal: Vec, - /// Hex nonce both halves answer. - pub nonce: String, - /// Parsed appraisal claims. - pub claims: Vec, -} - /// Collect GPU-signed evidence for `nonce` without appraising it. /// /// Returns the `evidences` array alone, which is the shape `attest @@ -250,50 +232,6 @@ pub async fn collect_evidence(nonce: &[u8], timeout: Duration) -> Result<(String Ok((nonce, evidences)) } -/// Collect GPU-signed evidence and appraise those exact bytes. -/// -/// Two steps rather than one `attest` run so the caller gets evidence a third -/// party can check alongside the verdict, and so both provably describe the -/// same report: the appraisal reads back the collected bytes instead of asking -/// the GPU a second time. The SDK independently rejects an evidence file whose -/// nonce does not match the one passed here. -pub async fn collect_and_appraise( - nonce: &[u8], - proxy_url: Option<&str>, - timeout: Duration, -) -> Result { - if !available() { - bail!("nvattest is not available in this image"); - } - let (nonce, evidences) = collect_evidence(nonce, timeout).await?; - - let dir = tempfile::tempdir().context("failed to create a directory for GPU evidence")?; - let path = dir.path().join("gpu-evidence.json"); - std::fs::write(&path, &evidences).context("failed to stage GPU evidence")?; - let path = path.to_str().context("GPU evidence path is not UTF-8")?; - - let mut args = args(&nonce, proxy_url)?; - args.extend([ - "--gpu-evidence-source".to_string(), - "file".to_string(), - "--gpu-evidence-file".to_string(), - path.to_string(), - ]); - let borrowed = args.iter().map(String::as_str).collect::>(); - let output = run_command(NVATTEST, &borrowed, timeout).await?; - if !output.stderr.is_empty() { - info!("nvattest: {}", truncated_lossy(&output.stderr, 2048)); - } - check_status(&output)?; - let claims = check_nonce(&output.stdout, &nonce)?; - Ok(CollectedAttestation { - evidence: String::from_utf8(evidences).context("GPU evidence is not valid UTF-8")?, - appraisal: output.stdout, - nonce, - claims, - }) -} - fn check_nonce_len(nonce: &[u8]) -> Result { if nonce.len() != NONCE_LEN { bail!( @@ -418,26 +356,6 @@ mod tests { ); } - #[test] - fn appraisal_args_read_back_the_collected_evidence() { - // The appraisal must consume the collected bytes rather than ask the - // GPU again, or the two halves of the response could describe different - // reports. Pin the flags that make that true. - let mut args = args("aa", None).unwrap(); - args.extend([ - "--gpu-evidence-source".to_string(), - "file".to_string(), - "--gpu-evidence-file".to_string(), - "/tmp/e.json".to_string(), - ]); - assert!(args - .windows(2) - .any(|w| w == ["--gpu-evidence-source", "file"])); - assert!(args - .windows(2) - .any(|w| w == ["--gpu-evidence-file", "/tmp/e.json"])); - } - #[tokio::test] async fn rejects_a_nonce_of_the_wrong_length() { let err = attest(&[0u8; 16], None, DEFAULT_TIMEOUT) diff --git a/sdk/curl/api.md b/sdk/curl/api.md index a2b53eea3..e4e0a6aa8 100644 --- a/sdk/curl/api.md +++ b/sdk/curl/api.md @@ -262,40 +262,7 @@ curl --unix-socket /var/run/dstack.sock http://dstack/Attest?report_data=0000000 ### 7. Attest GPU -Runs NVIDIA GPU attestation **now**, against a nonce you choose. Unlike -[`GpuInfo`](#8-gpu-info), which replays a record written at boot, this samples the -device at the moment of the call. - -Use it after anything that may have reinitialised the GPU. A driver reload leaves a -device that still answers NVML but can no longer attest, and this is how an -application detects that before submitting work. - -> [!IMPORTANT] -> `evidence` is checkable by anyone; `appraisal` is not. A relying party outside this -> CVM should verify `evidence` with its own verifier and ignore `appraisal`. -> -> **`evidence`** is `nvattest collect-evidence` output: one entry per device with the -> base64 SPDM attestation report and its certificate chain, signed by the GPU over the -> nonce you sent. To check it: verify the chain to NVIDIA's device-identity root, -> verify the report signature with the leaf key, confirm the nonce inside the report is -> the one you issued, then compare the measurements against NVIDIA's RIM documents. -> -> **`appraisal`** is the local verifier's verdict on exactly those bytes, provided -> because a caller inside the CVM is in the agent's trust domain and usually just wants -> the answer. It does not travel: its detached EAT is `alg:none` issued by -> `NVAT-LOCAL-VERIFIER`, and a claim like -> `x-nvidia-gpu-attestation-report-signature-verified: true` is an assertion about a -> check already performed, not proof anyone can redo. - -> [!WARNING] -> Neither field binds the GPU to *this* CVM. An NVIDIA report binds the device and the -> nonce and nothing else, so it can be relayed from a genuine remote GPU; deriving the -> nonce from a TDX quote does not help, because the relay can derive it too. Only -> TDISP/TEE-IO closes this, and no current Hopper/Blackwell deployment offers it. -> -> For evidence that the GPU is bound to this TD, use the boot-time `gpu-attestation` -> runtime event: measured dstack code (pinned by `os_image_hash`) appraised the GPU -> before any workload existed, with the digest in RTMR3 under the quote. +Collects vendor-native GPU evidence for a caller-chosen 32-byte nonce. **Endpoint:** `/AttestGpu` @@ -303,33 +270,24 @@ application detects that before submitting work. | Field | Type | Description | Example | |-------|------|-------------|----------| -| `nonce` | string | Exactly 32 bytes, hex-encoded, passed to the GPU verbatim. To bind a longer challenge, hash it yourself. | `"ab...ab"` (64 hex chars) | - -**Example:** -```bash -curl --unix-socket /var/run/dstack.sock -X POST \ - http://dstack/AttestGpu \ - -H 'Content-Type: application/json' \ - -d '{ - "nonce": "abababababababababababababababababababababababababababababababab" - }' -``` +| `nonce` | string | Exactly 32 bytes, hex-encoded and passed to the GPU verbatim. | `"ab...ab"` (64 hex chars) | **Response:** + ```json { - "evidence": "[{\"arch\": \"HOPPER\", \"nonce\": \"abab...\", \"evidence\": \"\", \"certificate\": \"\"}]", - "appraisal": "{\"result_code\": 0, \"claims\": [...]}", - "nonce": "abababababababababababababababababababababababababababababababab" + "bundles": [{ + "vendor": "nvidia", + "format": "nvidia-nvattest-collect-evidence-json-v1", + "evidence": "" + }] } ``` -The two halves describe the same report: dstack collects the evidence once and appraises -those exact bytes rather than asking the GPU twice. dstack has already checked that -nvattest succeeded and that every claim answers your nonce. Calls are serialised and rate-limited (one -attestation per 10s), because each one spawns `nvattest` and fetches OCSP and RIM -collateral from NVIDIA. A call arriving inside the cooldown is rejected with a wait -hint rather than queued. +Select a verifier using each bundle's `vendor` and `format`. The verifier must check +the evidence signature, certificate chain, measurements, and embedded nonce. The +agent does not appraise the evidence. Evidence does not by itself bind the GPU to this +CVM. ### 8. GPU Info diff --git a/sdk/go/README.md b/sdk/go/README.md index 8d2606514..6c1b8d147 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -577,27 +577,22 @@ instead. ##### `AttestGpu(ctx context.Context, nonce []byte) (*AttestGpuResponse, error)` -Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. Use it after -anything that may have reinitialised the GPU — a driver reload leaves a device that -answers NVML but can no longer attest. +Collects vendor-native GPU evidence for a caller-chosen 32-byte nonce. ```go resp, err := client.AttestGpu(ctx, nonce) if err != nil { - log.Fatal(err) + log.Fatal(err) +} +for _, bundle := range resp.Bundles { + fmt.Println(bundle.Vendor, bundle.Format, bundle.Evidence) } -fmt.Println(resp.Evidence) -``` - -> [!IMPORTANT] -> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its -> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on -> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should -> appraise `evidence` itself and ignore `appraisal`. -> -> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and -> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the -> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. +``` + +Select a verifier using each bundle's `Vendor` and `Format`. The verifier must check +the evidence signature, certificate chain, measurements, and embedded nonce. Evidence +is opaque and hex-encoded by the JSON RPC. It does not by itself bind the GPU to this +CVM. ##### `GpuInfo(ctx context.Context) (*GpuInfoResponse, error)` diff --git a/sdk/go/dstack/client.go b/sdk/go/dstack/client.go index 724f99781..e1c71c8ca 100644 --- a/sdk/go/dstack/client.go +++ b/sdk/go/dstack/client.go @@ -118,29 +118,16 @@ type GpuInfoResponse struct { Attestation string `json:"attestation"` } -// AttestGpuResponse is the result of a fresh, on-demand NVIDIA GPU attestation. -// -// Meaningful to a caller inside this CVM: the agent ran NVIDIA's verifier, which -// checked the GPU's report signature, its certificate chain and OCSP status, and the -// driver and VBIOS RIM signatures, all against this nonce. -// -// NOT independently verifiable by a third party, for two separate reasons. First, it -// is unsigned: --verifier local returns the verifier's conclusion, not the GPU's -// signed report. Its detached EAT is alg:none issued by NVAT-LOCAL-VERIFIER, and a -// claim like x-nvidia-gpu-attestation-report-signature-verified is an assertion, not -// proof; the signed artifacts are consumed during verification and not carried here. -// Second, even signed it would bind the device and the nonce but not the TD, so it is -// relayable from a genuine remote GPU. Use it as a local health check. For remote -// evidence use the boot-time `gpu-attestation` event, which measured code emits -// before any workload exists and which the event log binds to the quote. +// AttestGpuResponse is the result of fresh, on-demand GPU evidence collection. type AttestGpuResponse struct { - // Evidence is GPU-signed: collect-evidence output, one entry per device - // carrying the base64 SPDM attestation report and its certificate chain. + Bundles []GpuEvidenceBundle `json:"bundles"` +} + +type GpuEvidenceBundle struct { + Vendor string `json:"vendor"` + Format string `json:"format"` + // Evidence contains hex-encoded opaque bytes, as represented by the JSON RPC. Evidence string `json:"evidence"` - // Appraisal is the local verifier's verdict on exactly those bytes. Unsigned. - Appraisal string `json:"appraisal"` - // Nonce is the nonce both halves answer, hex-encoded, as it appears in eat_nonce. - Nonce string `json:"nonce"` } // Represents an event log entry in the TCB info diff --git a/sdk/go/dstack/client_test.go b/sdk/go/dstack/client_test.go index 380dfc073..5a3cc1bf4 100644 --- a/sdk/go/dstack/client_test.go +++ b/sdk/go/dstack/client_test.go @@ -94,10 +94,10 @@ func TestAttestGpu(t *testing.T) { t.Fatalf("nonce was not forwarded verbatim, got: %v", payload["nonce"]) } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]string{ - "evidence": evidence, - "appraisal": `{"result_code":0,"claims":[]}`, - "nonce": hex.EncodeToString(nonce), + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "bundles": []map[string]string{{ + "vendor": "nvidia", "format": "nvidia-test-v1", "evidence": evidence, + }}, }) })) defer server.Close() @@ -107,14 +107,8 @@ func TestAttestGpu(t *testing.T) { if err != nil { t.Fatal(err) } - if resp.Evidence != evidence { - t.Fatalf("unexpected evidence: %s", resp.Evidence) - } - if resp.Appraisal == "" { - t.Fatal("expected the appraisal to be carried alongside the evidence") - } - if resp.Nonce != hex.EncodeToString(nonce) { - t.Fatalf("unexpected nonce: %s", resp.Nonce) + if len(resp.Bundles) != 1 || resp.Bundles[0].Vendor != "nvidia" || resp.Bundles[0].Evidence != evidence { + t.Fatalf("unexpected evidence bundles: %+v", resp.Bundles) } } diff --git a/sdk/js/README.md b/sdk/js/README.md index a0063a335..f06eeb60a 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -97,23 +97,19 @@ const { attestation } = await client.attest('app-state-snapshot') ### `attestGpu(nonce)` -Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. Use it after -anything that may have reinitialised the GPU — a driver reload leaves a device that -answers NVML but can no longer attest. +Collects vendor-native GPU evidence for a caller-chosen 32-byte nonce. ```typescript -const { evidence } = await client.attestGpu(crypto.randomBytes(32)) +const { bundles } = await client.attestGpu(crypto.randomBytes(32)) +for (const bundle of bundles) { + console.log(bundle.vendor, bundle.format, bundle.evidence) +} ``` -> [!IMPORTANT] -> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its -> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on -> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should -> appraise `evidence` itself and ignore `appraisal`. -> -> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and -> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the -> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. +Select a verifier using each bundle's `vendor` and `format`. The verifier must check +the evidence signature, certificate chain, measurements, and embedded nonce. Evidence +is opaque and hex-encoded by the JSON RPC. It does not by itself bind the GPU to this +CVM. ### `gpuInfo()` diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index 0f1924a22..53393cb87 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -104,36 +104,18 @@ export interface AttestResponse { } /** - * Result of a fresh, on-demand NVIDIA GPU attestation. - * - * `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report - * and its certificate chain, per device, for the nonce you sent. A relying party - * verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce - * inside the report, and compares measurements against NVIDIA's RIM documents, using - * its own verifier and trusting nothing the CVM says. - * - * `appraisal` is the local verifier's verdict on those same bytes -- convenient inside - * the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, - * so a remote party should ignore it and appraise `evidence` itself. - * - * Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and - * nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would - * close that. + * Result of fresh, on-demand GPU evidence collection. */ export interface AttestGpuResponse { __name__: Readonly<'AttestGpuResponse'> + bundles: GpuEvidenceBundle[] +} - /** - * GPU-signed evidence: `collect-evidence` output, one entry per device carrying - * the base64 SPDM attestation report and its certificate chain. - */ - evidence: string - - /** The local verifier's verdict on exactly those bytes. Unsigned. */ - appraisal: string - - /** The nonce both halves answer, hex-encoded, as it appears in `eat_nonce`. */ - nonce: string +export interface GpuEvidenceBundle { + vendor: string + format: string + /** Hex-encoded opaque evidence bytes, as represented by the JSON RPC. */ + evidence: Hex } export interface GpuInfoResponse { @@ -345,7 +327,7 @@ export class DstackClient { throw new Error(`Nonce must be exactly 32 bytes, got ${nonce.length}.`) } const payload = JSON.stringify({ nonce: to_hex(nonce) }) - const result = await send_rpc_request<{ evidence: string, appraisal: string, nonce: string }>(this.endpoint, '/AttestGpu', payload) + const result = await send_rpc_request<{ bundles: GpuEvidenceBundle[] }>(this.endpoint, '/AttestGpu', payload) if ('error' in (result as any)) { throw new Error((result as any)['error'] as string) } diff --git a/sdk/python/README.md b/sdk/python/README.md index f7cb45df8..59accc958 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -102,24 +102,19 @@ print(result.decode_attestation()) # bytes ### On-demand GPU Attestation -`attest_gpu(nonce)` runs NVIDIA GPU attestation now, against a 32-byte nonce you -choose. Use it after anything that may have reinitialised the GPU — a driver reload -leaves a device that answers NVML but can no longer attest. +`attest_gpu(nonce)` collects vendor-native GPU evidence for a caller-chosen 32-byte +nonce. ```python result = client.attest_gpu(os.urandom(32)) -print(result.evidence) +for bundle in result.bundles: + print(bundle.vendor, bundle.format, bundle.evidence) ``` -> [!IMPORTANT] -> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its -> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on -> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should -> appraise `evidence` itself and ignore `appraisal`. -> -> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and -> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the -> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. +Select a verifier using each bundle's `vendor` and `format`. The verifier must check +the evidence signature, certificate chain, measurements, and embedded nonce. Evidence +is opaque and hex-encoded by the JSON RPC. It does not by itself bind the GPU to this +CVM. ### GPU Info diff --git a/sdk/python/src/dstack_sdk/__init__.py b/sdk/python/src/dstack_sdk/__init__.py index a0ac66144..42460a70d 100644 --- a/sdk/python/src/dstack_sdk/__init__.py +++ b/sdk/python/src/dstack_sdk/__init__.py @@ -12,6 +12,7 @@ from .dstack_client import GetTlsKeyResponse from .dstack_client import AttestGpuResponse from .dstack_client import GpuInfoResponse +from .dstack_client import GpuEvidenceBundle from .dstack_client import InfoResponse from .dstack_client import SignResponse from .dstack_client import TappdClient @@ -40,6 +41,7 @@ "GetTlsKeyResponse", "AttestResponse", "AttestGpuResponse", + "GpuEvidenceBundle", "GpuInfoResponse", "GetQuoteResponse", "InfoResponse", diff --git a/sdk/python/src/dstack_sdk/dstack_client.py b/sdk/python/src/dstack_sdk/dstack_client.py index 9ee27f8a8..554b0e7dd 100644 --- a/sdk/python/src/dstack_sdk/dstack_client.py +++ b/sdk/python/src/dstack_sdk/dstack_client.py @@ -157,27 +157,15 @@ def decode_attestation(self) -> bytes: return bytes.fromhex(self.attestation) -class AttestGpuResponse(BaseModel): - """Result of a fresh, on-demand NVIDIA GPU attestation. - - `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report - and its certificate chain, per device, for the nonce you sent. A relying party - verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce - inside the report, and compares measurements against NVIDIA's RIM documents, using - its own verifier and trusting nothing the CVM says. - - `appraisal` is the local verifier's verdict on those same bytes -- convenient inside - the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, - so a remote party should ignore it and appraise `evidence` itself. +class GpuEvidenceBundle(BaseModel): + vendor: str + format: str + evidence: str - Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and - nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would - close that. - """ - evidence: str - nonce: str - appraisal: str = "" +class AttestGpuResponse(BaseModel): + """Result of fresh, on-demand GPU evidence collection.""" + bundles: list[GpuEvidenceBundle] class GpuInfoResponse(BaseModel): @@ -464,21 +452,10 @@ async def attest( return AttestResponse(**result) async def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: - """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. - - `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report - and its certificate chain, per device, for the nonce you sent. A relying party - verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce - inside the report, and compares measurements against NVIDIA's RIM documents, using - its own verifier and trusting nothing the CVM says. - - `appraisal` is the local verifier's verdict on those same bytes -- convenient inside - the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, - so a remote party should ignore it and appraise `evidence` itself. + """Collect vendor-native GPU evidence for a 32-byte nonce. - Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and - nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would - close that. + Select a verifier using each bundle's vendor and format, and verify the + signature, certificate chain, measurements, and embedded nonce. """ if not isinstance(nonce, (bytes, bytearray)) or len(nonce) != 32: raise ValueError("nonce must be exactly 32 bytes") @@ -622,21 +599,10 @@ def attest( @call_async def attest_gpu(self, nonce: bytes) -> AttestGpuResponse: - """Run NVIDIA GPU attestation now, against a 32-byte nonce you choose. - - `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report - and its certificate chain, per device, for the nonce you sent. A relying party - verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce - inside the report, and compares measurements against NVIDIA's RIM documents, using - its own verifier and trusting nothing the CVM says. - - `appraisal` is the local verifier's verdict on those same bytes -- convenient inside - the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, - so a remote party should ignore it and appraise `evidence` itself. + """Collect vendor-native GPU evidence for a 32-byte nonce. - Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and - nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would - close that. + Select a verifier using each bundle's vendor and format, and verify the + signature, certificate chain, measurements, and embedded nonce. """ raise NotImplementedError diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index ef6534c37..1355104bc 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -125,21 +125,24 @@ async def test_async_client_attest(): @pytest.mark.asyncio async def test_async_client_attest_gpu(monkeypatch): evidence = '[{"arch":"HOPPER","evidence":"BASE64","certificate":"BASE64"}]' - appraisal = '{"result_code":0,"claims":[]}' nonce = bytes([0xAB]) * 32 async def fake_send(self, method, payload): assert method == "AttestGpu" assert payload == {"nonce": nonce.hex()} - return {"evidence": evidence, "appraisal": appraisal, "nonce": nonce.hex()} + return { + "bundles": [ + {"vendor": "nvidia", "format": "nvidia-test-v1", "evidence": evidence} + ] + } monkeypatch.setenv("DSTACK_SIMULATOR_ENDPOINT", "http://localhost:0") monkeypatch.setattr(AsyncDstackClient, "_send_rpc_request", fake_send) result = await AsyncDstackClient().attest_gpu(nonce) assert isinstance(result, AttestGpuResponse) - assert result.evidence == evidence - assert result.appraisal == appraisal - assert result.nonce == nonce.hex() + assert len(result.bundles) == 1 + assert result.bundles[0].vendor == "nvidia" + assert result.bundles[0].evidence == evidence @pytest.mark.asyncio diff --git a/sdk/rust/README.md b/sdk/rust/README.md index fbe0a8f12..a0326c119 100644 --- a/sdk/rust/README.md +++ b/sdk/rust/README.md @@ -106,24 +106,19 @@ Generates a versioned attestation with a custom 64-byte payload. #### `attest_gpu(nonce: Vec) -> AttestGpuResponse` -Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. Use it after -anything that may have reinitialised the GPU — a driver reload leaves a device that -answers NVML but can no longer attest. +Collects vendor-native GPU evidence for a caller-chosen 32-byte nonce. ```rust let result = client.attest_gpu(nonce.to_vec()).await?; -println!("{}", result.evidence); +for bundle in result.bundles { + println!("{} {} {}", bundle.vendor, bundle.format, bundle.evidence); +} ``` -> [!IMPORTANT] -> `evidence` is GPU-signed and checkable by anyone — the base64 SPDM report and its -> certificate chain, over your nonce. `appraisal` is the local verifier's verdict on -> those same bytes and does **not** travel (`alg:none` EAT), so a remote party should -> appraise `evidence` itself and ignore `appraisal`. -> -> Neither binds the GPU to this CVM: an NVIDIA report binds the device and nonce and -> nothing else, so it is relayable until TDISP/TEE-IO. For TD-bound evidence use the -> boot-time `gpu-attestation` event. Calls are rate-limited to one per 10s. +Select a verifier using each bundle's `vendor` and `format`. The verifier must check +the evidence signature, certificate chain, measurements, and embedded nonce. Evidence +is opaque and hex-encoded by the JSON RPC. It does not by itself bind the GPU to this +CVM. #### `gpu_info() -> GpuInfoResponse` diff --git a/sdk/rust/src/dstack_client.rs b/sdk/rust/src/dstack_client.rs index 31e11e7d3..31ac30ec9 100644 --- a/sdk/rust/src/dstack_client.rs +++ b/sdk/rust/src/dstack_client.rs @@ -164,21 +164,10 @@ impl DstackClient { Ok(response) } - /// Runs NVIDIA GPU attestation now, against a 32-byte nonce you choose. + /// Collects vendor-native GPU evidence for a caller-chosen 32-byte nonce. /// - /// `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report - /// and its certificate chain, per device, for the nonce you sent. A relying party - /// verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce - /// inside the report, and compares measurements against NVIDIA's RIM documents, using - /// its own verifier and trusting nothing the CVM says. - /// - /// `appraisal` is the local verifier's verdict on those same bytes -- convenient inside - /// the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, - /// so a remote party should ignore it and appraise `evidence` itself. - /// - /// Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and - /// nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would - /// close that. + /// Select a verifier using each bundle's vendor and format. The verifier must + /// check the signature, certificate chain, measurements, and embedded nonce. pub async fn attest_gpu(&self, nonce: Vec) -> Result { if nonce.len() != 32 { anyhow::bail!("Nonce must be exactly 32 bytes") diff --git a/sdk/rust/types/src/dstack.rs b/sdk/rust/types/src/dstack.rs index 13804c124..c2bf62bff 100644 --- a/sdk/rust/types/src/dstack.rs +++ b/sdk/rust/types/src/dstack.rs @@ -98,33 +98,25 @@ pub struct AttestResponse { pub attestation: String, } -/// Response from a fresh, on-demand NVIDIA GPU attestation. +/// Response from fresh, on-demand GPU evidence collection. /// -/// `evidence` is GPU-signed and checkable by anyone: the base64 SPDM attestation report -/// and its certificate chain, per device, for the nonce you sent. A relying party -/// verifies the chain to NVIDIA's root, checks the report signature, confirms the nonce -/// inside the report, and compares measurements against NVIDIA's RIM documents, using -/// its own verifier and trusting nothing the CVM says. -/// -/// `appraisal` is the local verifier's verdict on those same bytes -- convenient inside -/// the CVM, but not evidence: its detached EAT is `alg:none` from `NVAT-LOCAL-VERIFIER`, -/// so a remote party should ignore it and appraise `evidence` itself. -/// -/// Neither binds the GPU to this TD. An NVIDIA report binds the device and the nonce and -/// nothing else, so it can be relayed from a genuine remote GPU; only TDISP/TEE-IO would -/// close that. #[derive(Debug, Serialize, Deserialize)] #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] #[cfg_attr(feature = "borsh_schema", derive(BorshSchema))] pub struct AttestGpuResponse { - /// GPU-signed evidence: `collect-evidence` output, one entry per device - /// carrying the base64 SPDM attestation report and its certificate chain. + pub bundles: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] +#[cfg_attr(feature = "borsh_schema", derive(BorshSchema))] +pub struct GpuEvidenceBundle { + /// Stable GPU vendor identifier. + pub vendor: String, + /// Vendor-specific evidence format and version. + pub format: String, + /// Hex-encoded opaque evidence bytes, as represented by the JSON RPC. pub evidence: String, - /// The local verifier's appraisal of exactly those bytes. Unsigned. - #[serde(default)] - pub appraisal: String, - /// The nonce both halves answer, hex-encoded, as it appears in `eat_nonce`. - pub nonce: String, } /// Response containing the complete NVIDIA GPU attestation output. From 083cf824a822bbfe277385c7fadb65951554087c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 00:02:39 -0700 Subject: [PATCH 6/7] docs(guest-agent): correct GPU attestor comment --- dstack/guest-agent/src/rpc_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index 41bf3987f..51550913f 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -79,7 +79,7 @@ struct AppStateInner { platform: Arc, /// Present only when the app opted into health gating; see `health`. health: Option>, - /// Serialises and rate-limits on-demand GPU attestation. + /// Serialises on-demand GPU attestation. gpu_attestor: crate::gpu_attest::GpuAttestor, } From 3c2c3a339850f679ba42faf458288a84ff7ce5c7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Aug 2026 00:24:22 -0700 Subject: [PATCH 7/7] style(python): apply ruff fixes --- sdk/python/src/dstack_sdk/__init__.py | 4 ++-- sdk/python/src/dstack_sdk/dstack_client.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/src/dstack_sdk/__init__.py b/sdk/python/src/dstack_sdk/__init__.py index 42460a70d..5052772b2 100644 --- a/sdk/python/src/dstack_sdk/__init__.py +++ b/sdk/python/src/dstack_sdk/__init__.py @@ -4,15 +4,15 @@ from .dstack_client import AsyncDstackClient from .dstack_client import AsyncTappdClient +from .dstack_client import AttestGpuResponse from .dstack_client import AttestResponse from .dstack_client import DstackClient from .dstack_client import EventLog from .dstack_client import GetKeyResponse from .dstack_client import GetQuoteResponse from .dstack_client import GetTlsKeyResponse -from .dstack_client import AttestGpuResponse -from .dstack_client import GpuInfoResponse from .dstack_client import GpuEvidenceBundle +from .dstack_client import GpuInfoResponse from .dstack_client import InfoResponse from .dstack_client import SignResponse from .dstack_client import TappdClient diff --git a/sdk/python/src/dstack_sdk/dstack_client.py b/sdk/python/src/dstack_sdk/dstack_client.py index 2e262210a..aa5919d1b 100644 --- a/sdk/python/src/dstack_sdk/dstack_client.py +++ b/sdk/python/src/dstack_sdk/dstack_client.py @@ -171,6 +171,7 @@ class GpuEvidenceBundle(BaseModel): class AttestGpuResponse(BaseModel): """Result of fresh, on-demand GPU evidence collection.""" + bundles: list[GpuEvidenceBundle]