From 0797ebee9401d52f1907d872b8dd1bf4af0f9d0b Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 06:31:00 +0000 Subject: [PATCH 1/4] fix(attestation): hash report data before GetQuote (TC-963) --- CHANGELOG.md | 2 + docs/attestation.md | 53 +++++ tinycloud-node-server/src/dstack.rs | 208 ++++++++++++++++-- tinycloud-node-server/src/lib.rs | 2 +- .../src/routes/attestation.rs | 165 +++++++++++--- tinycloud-node-server/src/tee.rs | 7 +- 6 files changed, 382 insertions(+), 55 deletions(-) create mode 100644 docs/attestation.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b6cb28..944d6dca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +- `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. When the TEE can't produce a quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. Verifiers recompute the digest from their own nonce and expected DID; see `docs/attestation.md` (TC-963). + ## [1.20.7] - 2026-10-14 A security hardening release (TC-1132, #292) for webhook, quota and tunnel egress. diff --git a/docs/attestation.md b/docs/attestation.md new file mode 100644 index 00000000..9c9062ad --- /dev/null +++ b/docs/attestation.md @@ -0,0 +1,53 @@ +# `/attestation` response and verification + +`GET /attestation?nonce=` returns a fresh TDX quote from the dstack +guest agent when the node runs in a dstack TEE (`--features dstack`). + +## Response + +```json +{ + "mode": "dstack", + "quote": "", + "event_log": "", + "compose_hash": "", + "app_id": "", + "enforcer_did": "did:key:z6Mk...", + "key_binding": "xyz.tinycloud.share/tee-key/v1\\0:did:key:z6Mk...:", + "key_binding_sha256": "<64 hex chars>", + "timestamp": "" +} +``` + +- `key_binding` is the binding preimage: + `xyz.tinycloud.share/tee-key/v1\0::`. The `\0` is two + characters, a backslash and a zero (written `\\0` in JSON), not a NUL + byte. Without a `nonce` query parameter the string ends in `:`. +- `key_binding_sha256` is the hex SHA-256 of the UTF-8 bytes of + `key_binding`. The node sends these 32 bytes to dstack `GetQuote`, so the + quote's 64-byte REPORTDATA is the digest followed by 32 zero bytes. +- If the node can't produce a quote, or the guest agent reports quoting + different report data, the endpoint returns HTTP 503 with a + `quote generation failed: ...` message. Earlier releases had no + `key_binding_sha256` and answered that case with 200 and + `mode: "classic"`. +- A node outside a TEE still returns 200 with `mode: "classic"`. + +## Verifying + +1. Choose a fresh, unpredictable nonce and request + `/attestation?nonce=`. +2. Build the expected preimage yourself from the DID you expect and your + nonce: `xyz.tinycloud.share/tee-key/v1\0::`. Check + that it equals `key_binding` and that `enforcer_did` is the expected DID. +3. Compute SHA-256 of the preimage's UTF-8 bytes. It must equal + `key_binding_sha256`. +4. Verify the quote (for example with dstack-verifier or Intel DCAP) and + read its REPORTDATA from the verified quote, not from the response JSON. + REPORTDATA must be the digest followed by 32 zero bytes. +5. Replay `event_log` against the quote's RTMRs and check `compose_hash` and + `app_id` against the measurements you trust. + +The digest only binds what the verifier recomputes in steps 2 and 3, so +always derive the preimage from your own nonce and expected DID rather than +trusting the returned string. diff --git a/tinycloud-node-server/src/dstack.rs b/tinycloud-node-server/src/dstack.rs index 9d39fef4..387e4bcd 100644 --- a/tinycloud-node-server/src/dstack.rs +++ b/tinycloud-node-server/src/dstack.rs @@ -11,7 +11,10 @@ use tokio::net::UnixStream; const DEFAULT_SOCKET: &str = "/var/run/dstack.sock"; -fn socket_path() -> String { +/// TDX REPORTDATA is 64 bytes; the dstack guest agent rejects anything longer. +pub const MAX_REPORT_DATA_BYTES: usize = 64; + +pub fn socket_path() -> String { std::env::var("DSTACK_SIMULATOR_ENDPOINT").unwrap_or_else(|_| DEFAULT_SOCKET.to_string()) } @@ -26,8 +29,9 @@ struct GetKeyResponse { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GetQuoteResponse { - pub quote: String, // hex-encoded TDX quote - pub event_log: String, // hex-encoded event log + pub quote: String, // hex-encoded TDX quote + pub event_log: String, // JSON event log + pub report_data: String, // hex-encoded 64-byte REPORTDATA the agent quoted } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -41,18 +45,34 @@ pub struct InfoResponse { // Raw HTTP helpers // --------------------------------------------------------------------------- -/// Parse the body from a raw HTTP/1.1 response (everything after `\r\n\r\n`). +/// Return the body of a raw HTTP/1.1 response (everything after `\r\n\r\n`), +/// or an error naming the status if the guest agent did not answer 2xx. fn parse_response_body(raw: &[u8]) -> Result<&[u8]> { let header_end = raw .windows(4) .position(|w| w == b"\r\n\r\n") .ok_or_else(|| anyhow!("invalid HTTP response: missing header terminator"))?; - Ok(&raw[header_end + 4..]) + let status = raw[..header_end] + .split(|&b| b == b' ') + .nth(1) + .and_then(|code| std::str::from_utf8(code).ok()?.parse::().ok()) + .ok_or_else(|| anyhow!("invalid HTTP response: missing status code"))?; + let body = &raw[header_end + 4..]; + if !(200..300).contains(&status) { + // The error body stays in the operator log; callers see only the status. + let excerpt = String::from_utf8_lossy(&body[..body.len().min(512)]); + tracing::warn!(status, body = %excerpt, "dstack guest agent returned an error"); + anyhow::bail!("guest agent returned HTTP {status}"); + } + Ok(body) } -async fn post_json(path: &str, body: &serde_json::Value) -> Result { - let socket = socket_path(); - let mut stream = UnixStream::connect(&socket) +async fn post_json( + socket: &str, + path: &str, + body: &serde_json::Value, +) -> Result { + let mut stream = UnixStream::connect(socket) .await .with_context(|| format!("connecting to dstack socket at {socket}"))?; @@ -77,9 +97,8 @@ async fn post_json(path: &str, body: &serde_json::Value) -> Result Result { - let socket = socket_path(); - let mut stream = UnixStream::connect(&socket) +async fn get_json(socket: &str, path: &str) -> Result { + let mut stream = UnixStream::connect(socket) .await .with_context(|| format!("connecting to dstack socket at {socket}"))?; @@ -110,7 +129,7 @@ async fn get_json(path: &str) -> Result { pub async fn get_key(path: &str) -> Result> { let body = serde_json::json!({ "path": path }); let resp: GetKeyResponse = serde_json::from_value( - post_json("/GetKey", &body) + post_json(&socket_path(), "/GetKey", &body) .await .context("dstack GetKey request")?, ) @@ -118,24 +137,175 @@ pub async fn get_key(path: &str) -> Result> { hex::decode(&resp.key).context("decoding hex key bytes from dstack") } -/// Request a TDX attestation quote from the TEE. -pub async fn get_quote(report_data: &[u8]) -> Result { +/// Request a TDX attestation quote from the guest agent at `socket`. +pub async fn get_quote(socket: &str, report_data: &[u8]) -> Result { + anyhow::ensure!( + report_data.len() <= MAX_REPORT_DATA_BYTES, + "report_data is {} bytes; a TDX quote holds at most {MAX_REPORT_DATA_BYTES}", + report_data.len() + ); let body = serde_json::json!({ "report_data": hex::encode(report_data) }); - serde_json::from_value( - post_json("/GetQuote", &body) + let response: GetQuoteResponse = serde_json::from_value( + post_json(socket, "/GetQuote", &body) .await .context("dstack GetQuote request")?, ) - .context("parsing GetQuote response") + .context("parsing GetQuote response")?; + + // dstack zero-pads report_data to 64 bytes; refuse a quote over anything else. + let mut expected = report_data.to_vec(); + expected.resize(MAX_REPORT_DATA_BYTES, 0); + let quoted = hex::decode(response.report_data.trim_start_matches("0x")) + .context("decoding GetQuote report_data")?; + anyhow::ensure!( + quoted == expected, + "GetQuote report_data does not match the requested report_data" + ); + Ok(response) } /// Retrieve TEE instance information. pub async fn get_info() -> Result { - serde_json::from_value(get_json("/Info").await.context("dstack Info request")?) - .context("parsing Info response") + serde_json::from_value( + get_json(&socket_path(), "/Info") + .await + .context("dstack Info request")?, + ) + .context("parsing Info response") } /// Check whether the dstack socket is reachable. pub fn is_available() -> bool { std::path::Path::new(&socket_path()).exists() } + +/// A one-shot stand-in for the dstack guest agent on a temporary Unix socket. +#[cfg(test)] +pub(crate) mod stub { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::UnixListener; + use tokio::task::JoinHandle; + + pub(crate) struct Agent { + _dir: tempfile::TempDir, + pub(crate) socket: String, + /// Resolves to the raw request the agent received. + pub(crate) request: JoinHandle>, + } + + /// Accept one connection, capture the request, then reply with + /// `status_line` (e.g. `"400 Bad Request"`) and the JSON `body`. + pub(crate) fn serve_once(status_line: &str, body: &str) -> Agent { + let (status_line, body) = (status_line.to_owned(), body.to_owned()); + serve(move |_| (status_line, body)) + } + + /// Answer one GetQuote like dstack: a quote whose `report_data` is the + /// requested bytes zero-padded to 64. + pub(crate) fn serve_quote() -> Agent { + serve(|request| { + let mut padded = requested_report_data(request); + padded.resize(super::MAX_REPORT_DATA_BYTES, 0); + let body = serde_json::json!({ + "quote": "aa", + "event_log": "[]", + "report_data": hex::encode(padded), + }); + ("200 OK".to_owned(), body.to_string()) + }) + } + + fn serve(reply: impl FnOnce(&[u8]) -> (String, String) + Send + 'static) -> Agent { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("dstack.sock").to_str().unwrap().to_owned(); + let listener = UnixListener::bind(&socket).unwrap(); + let request = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + stream.read_to_end(&mut request).await.unwrap(); + let (status_line, body) = reply(&request); + let response = format!( + "HTTP/1.1 {status_line}\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + stream.shutdown().await.unwrap(); + request + }); + Agent { + _dir: dir, + socket, + request, + } + } + + /// The decoded `report_data` bytes of a captured GetQuote request. + pub(crate) fn requested_report_data(raw: &[u8]) -> Vec { + let split = raw.windows(4).position(|w| w == b"\r\n\r\n").unwrap(); + let body: serde_json::Value = serde_json::from_slice(&raw[split + 4..]).unwrap(); + hex::decode(body["report_data"].as_str().unwrap()).unwrap() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn get_quote_sends_exactly_64_bytes_of_report_data() { + let agent = stub::serve_quote(); + let quote = get_quote(&agent.socket, &[0x42; MAX_REPORT_DATA_BYTES]) + .await + .unwrap(); + assert_eq!(quote.quote, "aa"); + assert_eq!(quote.event_log, "[]"); + let sent = stub::requested_report_data(&agent.request.await.unwrap()); + assert_eq!(sent, vec![0x42; MAX_REPORT_DATA_BYTES]); + } + + #[tokio::test] + async fn get_quote_refuses_65_bytes_without_contacting_dstack() { + let dir = tempfile::tempdir().unwrap(); + let absent = dir.path().join("absent.sock"); + let err = get_quote(absent.to_str().unwrap(), &[0x42; MAX_REPORT_DATA_BYTES + 1]) + .await + .unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("65 bytes"), "{message}"); + assert!(!message.contains("connecting"), "{message}"); + } + + #[tokio::test] + async fn get_quote_reports_guest_agent_error_status_not_a_parse_failure() { + let agent = stub::serve_once( + "400 Bad Request", + r#"{"error":"report_data must be at most 64 bytes"}"#, + ); + let err = get_quote(&agent.socket, &[0x42; 32]).await.unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("HTTP 400"), "{message}"); + assert!(!message.contains("parsing"), "{message}"); + agent.request.await.unwrap(); + } + + #[tokio::test] + async fn get_quote_rejects_a_quote_over_different_report_data() { + let other = hex::encode([0x24u8; MAX_REPORT_DATA_BYTES]); + let agent = stub::serve_once( + "200 OK", + &format!(r#"{{"quote":"aa","event_log":"[]","report_data":"{other}"}}"#), + ); + let err = get_quote(&agent.socket, &[0x42; 32]).await.unwrap_err(); + assert!(format!("{err:#}").contains("report_data"), "{err:#}"); + agent.request.await.unwrap(); + } + + #[tokio::test] + async fn get_quote_rejects_a_quote_without_report_data() { + let agent = stub::serve_once("200 OK", r#"{"quote":"aa","event_log":"[]"}"#); + let err = get_quote(&agent.socket, &[0x42; 32]).await.unwrap_err(); + assert!(format!("{err:#}").contains("report_data"), "{err:#}"); + agent.request.await.unwrap(); + } +} diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 7e7b8def..5e63f530 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -28,7 +28,7 @@ pub mod allow_list; pub mod auth_guards; pub mod authorization; pub mod config; -#[cfg(feature = "dstack")] +#[cfg(any(test, feature = "dstack"))] pub mod dstack; pub mod egress; pub mod hooks; diff --git a/tinycloud-node-server/src/routes/attestation.rs b/tinycloud-node-server/src/routes/attestation.rs index 7b5673f3..3d23ac8b 100644 --- a/tinycloud-node-server/src/routes/attestation.rs +++ b/tinycloud-node-server/src/routes/attestation.rs @@ -1,3 +1,4 @@ +use rocket::http::Status; use rocket::serde::json::Json; use rocket::State; @@ -14,57 +15,153 @@ use crate::tee::{AttestationResponse, TeeContext}; /// In classic mode, returns a simple response indicating no TEE is available. /// /// An optional `nonce` parameter can be included to prevent replay attacks. +/// The quote's REPORTDATA carries SHA-256 of the returned `key_binding`, which +/// includes the enforcer DID and the nonce (see `docs/attestation.md`). +/// +/// Returns 503 if the TEE cannot produce a quote. #[get("/attestation?")] pub async fn attestation( tee: &State>, nonce: Option, -) -> Json { +) -> Result, (Status, String)> { match tee.inner() { Some(ctx) => { // In TEE mode, get a fresh quote from dstack #[cfg(feature = "dstack")] { - let key_binding = format!( - "xyz.tinycloud.share/tee-key/v1\\0:{}:{}", - ctx.enforcer_did, - nonce.as_deref().unwrap_or("") - ); - let report_data = key_binding.as_bytes().to_vec(); - - match crate::dstack::get_quote(&report_data).await { - Ok(quote_resp) => Json(AttestationResponse::Dstack { - quote: quote_resp.quote, - event_log: quote_resp.event_log, - compose_hash: ctx.compose_hash.clone(), - app_id: ctx.app_id.clone(), - enforcer_did: ctx.enforcer_did.clone(), - key_binding, - timestamp: time::OffsetDateTime::now_utc() - .format(&time::format_description::well_known::Rfc3339) - .unwrap_or_default(), - }), - Err(e) => { - tracing::error!("Failed to get TDX quote: {}", e); - Json(AttestationResponse::Classic { - message: format!( - "TEE context available but quote generation failed: {}", - e - ), - }) - } - } + let socket = crate::dstack::socket_path(); + dstack_attestation(&socket, ctx, nonce.as_deref()) + .await + .map(Json) + .map_err(|e| { + tracing::error!("Failed to get TDX quote: {:#}", e); + ( + Status::ServiceUnavailable, + format!("quote generation failed: {e:#}"), + ) + }) } #[cfg(not(feature = "dstack"))] { let _ = nonce; // suppress unused warning let _ = ctx; - Json(AttestationResponse::Classic { + Ok(Json(AttestationResponse::Classic { message: "TEE context available but dstack feature not compiled in".to_string(), - }) + })) } } - None => Json(AttestationResponse::Classic { + None => Ok(Json(AttestationResponse::Classic { message: "This instance is not running in a TEE".to_string(), - }), + })), + } +} + +/// Quote this instance's key binding through the dstack guest agent at `socket`. +#[cfg(any(test, feature = "dstack"))] +async fn dstack_attestation( + socket: &str, + ctx: &TeeContext, + nonce: Option<&str>, +) -> anyhow::Result { + use sha2::Digest as _; + + let key_binding = format!( + "xyz.tinycloud.share/tee-key/v1\\0:{}:{}", + ctx.enforcer_did, + nonce.unwrap_or("") + ); + // REPORTDATA holds 64 bytes, so the quote carries the binding's SHA-256 + // and the response returns the preimage for verifiers to recompute. + let report_data = sha2::Sha256::digest(key_binding.as_bytes()); + + let quote_resp = crate::dstack::get_quote(socket, &report_data).await?; + Ok(AttestationResponse::Dstack { + quote: quote_resp.quote, + event_log: quote_resp.event_log, + compose_hash: ctx.compose_hash.clone(), + app_id: ctx.app_id.clone(), + enforcer_did: ctx.enforcer_did.clone(), + key_binding, + key_binding_sha256: hex::encode(report_data), + timestamp: time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_default(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dstack::stub; + use sha2::Digest; + + // A real node did:key; with the domain tag alone the raw binding is + // already past dstack's 64-byte REPORTDATA limit. + const DID: &str = "did:key:z6MkqGC3nWZhYieEVTVDKW5v588CiGfsDSmRVG9ZwwWTvLSK"; + + fn ctx() -> TeeContext { + TeeContext { + app_id: "app".into(), + compose_hash: "compose".into(), + instance_id: "instance".into(), + enforcer_did: DID.into(), + } + } + + /// The response and the report_data the guest agent was asked to quote. + async fn attest(nonce: Option<&str>) -> (AttestationResponse, Vec) { + let agent = stub::serve_quote(); + let response = dstack_attestation(&agent.socket, &ctx(), nonce) + .await + .unwrap(); + let sent = stub::requested_report_data(&agent.request.await.unwrap()); + (response, sent) + } + + #[tokio::test] + async fn quote_request_carries_a_32_byte_binding_hash() { + let (_, sent) = attest(Some("a-verifier-chosen-nonce-of-some-length")).await; + assert_eq!(sent.len(), 32); + } + + #[tokio::test] + async fn each_nonce_is_bound_into_distinct_report_data() { + let (response, with_a) = attest(Some("nonce-a")).await; + let (_, with_b) = attest(Some("nonce-b")).await; + let (_, without) = attest(None).await; + assert_ne!(with_a, with_b); + assert_ne!(with_a, without); + let AttestationResponse::Dstack { key_binding, .. } = response else { + panic!("expected a dstack attestation"); + }; + assert!(key_binding.ends_with(":nonce-a"), "{key_binding}"); + } + + #[tokio::test] + async fn verifier_recomputes_report_data_from_the_returned_preimage() { + let nonce = "verifier-nonce-1"; + let (response, sent) = attest(Some(nonce)).await; + let AttestationResponse::Dstack { + enforcer_did, + key_binding, + key_binding_sha256, + .. + } = response + else { + panic!("expected a dstack attestation"); + }; + // A verifier rebuilds the preimage from the DID it expects and its own nonce. + assert_eq!(enforcer_did, DID); + assert_eq!( + key_binding, + format!("xyz.tinycloud.share/tee-key/v1\\0:{DID}:{nonce}") + ); + let digest = sha2::Sha256::digest(key_binding.as_bytes()); + assert_eq!(key_binding_sha256, hex::encode(digest)); + // dstack zero-pads report_data into the quote's 64-byte REPORTDATA. + let mut report_data = sent; + report_data.resize(crate::dstack::MAX_REPORT_DATA_BYTES, 0); + assert_eq!(&report_data[..32], &digest[..]); + assert_eq!(&report_data[32..], &[0u8; 32]); } } diff --git a/tinycloud-node-server/src/tee.rs b/tinycloud-node-server/src/tee.rs index 3e311455..9071520c 100644 --- a/tinycloud-node-server/src/tee.rs +++ b/tinycloud-node-server/src/tee.rs @@ -57,8 +57,13 @@ pub enum AttestationResponse { app_id: String, /// DID whose public key is bound to this attested instance. enforcer_did: String, - /// The report-data binding input used for the quote. + /// The quote's binding preimage: + /// `xyz.tinycloud.share/tee-key/v1\0::`, + /// where `\0` is the two characters backslash and zero. key_binding: String, + /// Hex SHA-256 of `key_binding`'s UTF-8 bytes. The quote's 64-byte + /// REPORTDATA is these 32 bytes followed by 32 zero bytes. + key_binding_sha256: String, /// ISO 8601 timestamp timestamp: String, }, From 4555b7c66c2160f31cd0d1c9b47a37b4895b49b2 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 06:51:35 +0000 Subject: [PATCH 2/4] fix(attestation): check quote REPORTDATA and bound nonce and request rate (TC-963) --- CHANGELOG.md | 2 +- docs/attestation.md | 43 +++-- tinycloud-node-server/src/dstack.rs | 132 ++++++++++++- .../src/routes/attestation.rs | 175 +++++++++++++++--- 4 files changed, 305 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 944d6dca..eaca379d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -- `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. When the TEE can't produce a quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. Verifiers recompute the digest from their own nonce and expected DID; see `docs/attestation.md` (TC-963). +- `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). ## [1.20.7] - 2026-10-14 diff --git a/docs/attestation.md b/docs/attestation.md index 9c9062ad..2ffdbef0 100644 --- a/docs/attestation.md +++ b/docs/attestation.md @@ -1,7 +1,17 @@ # `/attestation` response and verification `GET /attestation?nonce=` returns a fresh TDX quote from the dstack -guest agent when the node runs in a dstack TEE (`--features dstack`). +guest agent when the node runs in a dstack TEE (`--features dstack`). The +optional `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _`; anything else +gets 400. The endpoint shares the public routes' per-IP rate limit (429). + +**The node checks the binding is present; you must verify the quote.** Before +answering 200, the node parses the returned TDX v4 quote and requires its +REPORTDATA field (48-byte quote header, then TD report body offset 520; see +Intel's TDX DCAP Quoting Library API, Appendix A, "Version 4 Quote Format") +to equal `key_binding_sha256` followed by 32 zero bytes. This is a structural +check only. The node does not verify the quote's signature, certificate chain +or TCB status, so a 200 response proves nothing on its own. ## Response @@ -26,27 +36,34 @@ guest agent when the node runs in a dstack TEE (`--features dstack`). - `key_binding_sha256` is the hex SHA-256 of the UTF-8 bytes of `key_binding`. The node sends these 32 bytes to dstack `GetQuote`, so the quote's 64-byte REPORTDATA is the digest followed by 32 zero bytes. -- If the node can't produce a quote, or the guest agent reports quoting - different report data, the endpoint returns HTTP 503 with a - `quote generation failed: ...` message. Earlier releases had no - `key_binding_sha256` and answered that case with 200 and - `mode: "classic"`. -- A node outside a TEE still returns 200 with `mode: "classic"`. +- `timestamp` is the node's clock and is not covered by the quote. A quote + requested without a nonce proves no freshness and can be replayed. +- If the node can't produce a quote, or the quote's REPORTDATA (or the guest + agent's echo of it) is not the digest, or the quote is truncated or not a + TDX v4 quote, the endpoint returns HTTP 503 `quote generation failed`. + Earlier releases had no `key_binding_sha256` and answered quote failures + with 200 and `mode: "classic"`. +- A node outside a TEE returns 200 with `mode: "classic"`. A verifier that + expects a TEE must treat that as a failure. ## Verifying 1. Choose a fresh, unpredictable nonce and request - `/attestation?nonce=`. + `/attestation?nonce=`. Reject anything but 200 with + `mode: "dstack"`. 2. Build the expected preimage yourself from the DID you expect and your nonce: `xyz.tinycloud.share/tee-key/v1\0::`. Check that it equals `key_binding` and that `enforcer_did` is the expected DID. 3. Compute SHA-256 of the preimage's UTF-8 bytes. It must equal `key_binding_sha256`. -4. Verify the quote (for example with dstack-verifier or Intel DCAP) and - read its REPORTDATA from the verified quote, not from the response JSON. - REPORTDATA must be the digest followed by 32 zero bytes. -5. Replay `event_log` against the quote's RTMRs and check `compose_hash` and - `app_id` against the measurements you trust. +4. Verify the quote (for example with dstack-verifier or Intel DCAP), + including its TCB status, and read REPORTDATA from the verified quote, + not from the response JSON. REPORTDATA must be the digest followed by 32 + zero bytes. +5. Check the quote's MRTD and RTMR0-2 against the dstack OS image you trust. + Replay `event_log` against RTMR3 and take `compose-hash` and `app-id` from + the replayed events, not from the JSON fields; compare them with the + release you expect. The digest only binds what the verifier recomputes in steps 2 and 3, so always derive the preimage from your own nonce and expected DID rather than diff --git a/tinycloud-node-server/src/dstack.rs b/tinycloud-node-server/src/dstack.rs index 387e4bcd..6a135a0a 100644 --- a/tinycloud-node-server/src/dstack.rs +++ b/tinycloud-node-server/src/dstack.rs @@ -152,18 +152,63 @@ pub async fn get_quote(socket: &str, report_data: &[u8]) -> Result Result<&[u8]> { + let sig_len_at = QUOTE_HEADER_LEN + TD_QUOTE_BODY_LEN; + anyhow::ensure!( + quote.len() >= sig_len_at + 4, + "quote is truncated ({} bytes)", + quote.len() + ); + let version = u16::from_le_bytes([quote[0], quote[1]]); + let tee_type = u32::from_le_bytes([quote[4], quote[5], quote[6], quote[7]]); + anyhow::ensure!( + version == 4 && tee_type == TEE_TYPE_TDX, + "unsupported quote: version {version}, TEE type {tee_type:#x}" + ); + let sig_len = u32::from_le_bytes([ + quote[sig_len_at], + quote[sig_len_at + 1], + quote[sig_len_at + 2], + quote[sig_len_at + 3], + ]) as usize; + anyhow::ensure!( + quote.len() - (sig_len_at + 4) >= sig_len, + "quote signature data is truncated" + ); + Ok("e[TD_REPORT_DATA_OFFSET..TD_REPORT_DATA_OFFSET + MAX_REPORT_DATA_BYTES]) +} + /// Retrieve TEE instance information. pub async fn get_info() -> Result { serde_json::from_value( @@ -200,14 +245,35 @@ pub(crate) mod stub { serve(move |_| (status_line, body)) } - /// Answer one GetQuote like dstack: a quote whose `report_data` is the - /// requested bytes zero-padded to 64. + /// A structurally valid TDX v4 quote: 48-byte header (version 4, ECDSA-256 + /// key type 2, TEE type 0x81), 584-byte TD report body carrying + /// `report_data` at body offset 520, then a length-prefixed signature + /// section. The signature bytes are filler; nothing here verifies them. + pub(crate) fn tdx_quote(report_data: &[u8]) -> Vec { + let mut quote = vec![0u8; 48 + 584]; + quote[0..2].copy_from_slice(&4u16.to_le_bytes()); + quote[2..4].copy_from_slice(&2u16.to_le_bytes()); + quote[4..8].copy_from_slice(&0x81u32.to_le_bytes()); + quote[48 + 520..48 + 584].copy_from_slice(report_data); + quote.extend_from_slice(&64u32.to_le_bytes()); + quote.extend_from_slice(&[0x5a; 64]); + quote + } + + /// Answer one GetQuote like dstack: a quote over the requested bytes + /// zero-padded to 64, echoed back as `report_data`. pub(crate) fn serve_quote() -> Agent { + serve_quote_with(|padded| hex::encode(tdx_quote(padded))) + } + + /// Like [`serve_quote`], echoing the right `report_data` but answering + /// with the hex quote `make` builds from the padded request. + pub(crate) fn serve_quote_with(make: impl FnOnce(&[u8]) -> String + Send + 'static) -> Agent { serve(|request| { let mut padded = requested_report_data(request); padded.resize(super::MAX_REPORT_DATA_BYTES, 0); let body = serde_json::json!({ - "quote": "aa", + "quote": make(&padded), "event_log": "[]", "report_data": hex::encode(padded), }); @@ -258,7 +324,7 @@ mod tests { let quote = get_quote(&agent.socket, &[0x42; MAX_REPORT_DATA_BYTES]) .await .unwrap(); - assert_eq!(quote.quote, "aa"); + assert_eq!(quote.quote, hex::encode(stub::tdx_quote(&[0x42; 64]))); assert_eq!(quote.event_log, "[]"); let sent = stub::requested_report_data(&agent.request.await.unwrap()); assert_eq!(sent, vec![0x42; MAX_REPORT_DATA_BYTES]); @@ -292,9 +358,10 @@ mod tests { #[tokio::test] async fn get_quote_rejects_a_quote_over_different_report_data() { let other = hex::encode([0x24u8; MAX_REPORT_DATA_BYTES]); + let quote = hex::encode(stub::tdx_quote(&[0x24; 64])); let agent = stub::serve_once( "200 OK", - &format!(r#"{{"quote":"aa","event_log":"[]","report_data":"{other}"}}"#), + &format!(r#"{{"quote":"{quote}","event_log":"[]","report_data":"{other}"}}"#), ); let err = get_quote(&agent.socket, &[0x42; 32]).await.unwrap_err(); assert!(format!("{err:#}").contains("report_data"), "{err:#}"); @@ -303,9 +370,56 @@ mod tests { #[tokio::test] async fn get_quote_rejects_a_quote_without_report_data() { - let agent = stub::serve_once("200 OK", r#"{"quote":"aa","event_log":"[]"}"#); + let quote = hex::encode(stub::tdx_quote(&[0x42; 64])); + let agent = stub::serve_once( + "200 OK", + &format!(r#"{{"quote":"{quote}","event_log":"[]"}}"#), + ); let err = get_quote(&agent.socket, &[0x42; 32]).await.unwrap_err(); assert!(format!("{err:#}").contains("report_data"), "{err:#}"); agent.request.await.unwrap(); } + + /// get_quote over 32 bytes of 0x42 against an agent that echoes the right + /// report_data but answers with the quote `make` builds. + async fn quote_failure(make: impl FnOnce(&[u8]) -> String + Send + 'static) -> String { + let agent = stub::serve_quote_with(make); + let err = get_quote(&agent.socket, &[0x42; 32]).await.unwrap_err(); + agent.request.await.unwrap(); + format!("{err:#}") + } + + #[tokio::test] + async fn get_quote_rejects_a_quote_whose_reportdata_differs_from_a_matching_echo() { + let message = quote_failure(|_| hex::encode(stub::tdx_quote(&[0x24; 64]))).await; + assert!(message.contains("REPORTDATA"), "{message}"); + } + + #[tokio::test] + async fn get_quote_rejects_truncated_and_malformed_quotes() { + let makers: [fn(&[u8]) -> String; 5] = [ + // Cut inside REPORTDATA. + |padded: &[u8]| hex::encode(&stub::tdx_quote(padded)[..600]), + // Body complete, signature section missing. + |padded: &[u8]| hex::encode(&stub::tdx_quote(padded)[..632]), + // Signature length runs past the end. + |padded: &[u8]| { + let mut quote = stub::tdx_quote(padded); + quote.truncate(quote.len() - 1); + hex::encode(quote) + }, + // An SGX (TEE type 0) quote. + |padded: &[u8]| { + let mut quote = stub::tdx_quote(padded); + quote[4] = 0; + hex::encode(quote) + }, + // Not hex. + |_: &[u8]| "zz".to_owned(), + ]; + for make in makers { + let message = quote_failure(make).await; + assert!(message.contains("quote"), "{message}"); + } + } } diff --git a/tinycloud-node-server/src/routes/attestation.rs b/tinycloud-node-server/src/routes/attestation.rs index 3d23ac8b..f9910adf 100644 --- a/tinycloud-node-server/src/routes/attestation.rs +++ b/tinycloud-node-server/src/routes/attestation.rs @@ -2,8 +2,12 @@ use rocket::http::Status; use rocket::serde::json::Json; use rocket::State; +use crate::routes::public::{ClientIp, RateLimiter}; use crate::tee::{AttestationResponse, TeeContext}; +/// Longest accepted `nonce`, in bytes. +const MAX_NONCE_LEN: usize = 64; + /// Get attestation information about this server instance. /// /// In TEE mode (dstack), returns a TDX quote that can be verified @@ -14,32 +18,44 @@ use crate::tee::{AttestationResponse, TeeContext}; /// /// In classic mode, returns a simple response indicating no TEE is available. /// -/// An optional `nonce` parameter can be included to prevent replay attacks. -/// The quote's REPORTDATA carries SHA-256 of the returned `key_binding`, which +/// An optional `nonce` (1-64 characters of `A-Z a-z 0-9 - _`) binds the quote +/// to the caller's challenge; without one the quote proves no freshness. The +/// quote's REPORTDATA carries SHA-256 of the returned `key_binding`, which /// includes the enforcer DID and the nonce (see `docs/attestation.md`). +/// Before answering 200 the node checks that the quote's REPORTDATA field +/// holds that digest; it does not verify the quote's signature. /// -/// Returns 503 if the TEE cannot produce a quote. +/// Shares the per-IP limit of the public routes, since each call quotes. +/// Returns 400 for a malformed nonce and 503 if the TEE cannot produce a +/// quote bound to the digest. #[get("/attestation?")] pub async fn attestation( tee: &State>, + client_ip: ClientIp, + rate_limiter: &State, nonce: Option, ) -> Result, (Status, String)> { + rate_limiter + .check(client_ip.0) + .map_err(|s| (s, "Rate limit exceeded".to_string()))?; + if let Some(nonce) = nonce.as_deref() { + let well_formed = (1..=MAX_NONCE_LEN).contains(&nonce.len()) + && nonce + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'); + if !well_formed { + return Err(( + Status::BadRequest, + format!("nonce must be 1-{MAX_NONCE_LEN} characters of A-Z, a-z, 0-9, '-' or '_'"), + )); + } + } match tee.inner() { Some(ctx) => { // In TEE mode, get a fresh quote from dstack #[cfg(feature = "dstack")] { - let socket = crate::dstack::socket_path(); - dstack_attestation(&socket, ctx, nonce.as_deref()) - .await - .map(Json) - .map_err(|e| { - tracing::error!("Failed to get TDX quote: {:#}", e); - ( - Status::ServiceUnavailable, - format!("quote generation failed: {e:#}"), - ) - }) + dstack_attestation(&crate::dstack::socket_path(), ctx, nonce.as_deref()).await } #[cfg(not(feature = "dstack"))] { @@ -56,13 +72,14 @@ pub async fn attestation( } } -/// Quote this instance's key binding through the dstack guest agent at `socket`. +/// Quote this instance's key binding through the dstack guest agent at +/// `socket`. Any failure is a 503; the cause stays in the operator log. #[cfg(any(test, feature = "dstack"))] async fn dstack_attestation( socket: &str, ctx: &TeeContext, nonce: Option<&str>, -) -> anyhow::Result { +) -> Result, (Status, String)> { use sha2::Digest as _; let key_binding = format!( @@ -74,8 +91,16 @@ async fn dstack_attestation( // and the response returns the preimage for verifiers to recompute. let report_data = sha2::Sha256::digest(key_binding.as_bytes()); - let quote_resp = crate::dstack::get_quote(socket, &report_data).await?; - Ok(AttestationResponse::Dstack { + let quote_resp = crate::dstack::get_quote(socket, &report_data) + .await + .map_err(|e| { + tracing::error!("Failed to get TDX quote: {:#}", e); + ( + Status::ServiceUnavailable, + "quote generation failed".to_string(), + ) + })?; + Ok(Json(AttestationResponse::Dstack { quote: quote_resp.quote, event_log: quote_resp.event_log, compose_hash: ctx.compose_hash.clone(), @@ -86,7 +111,7 @@ async fn dstack_attestation( timestamp: time::OffsetDateTime::now_utc() .format(&time::format_description::well_known::Rfc3339) .unwrap_or_default(), - }) + })) } #[cfg(test)] @@ -113,11 +138,45 @@ mod tests { let agent = stub::serve_quote(); let response = dstack_attestation(&agent.socket, &ctx(), nonce) .await - .unwrap(); + .unwrap() + .into_inner(); let sent = stub::requested_report_data(&agent.request.await.unwrap()); (response, sent) } + #[tokio::test] + async fn guest_agent_error_is_a_503_not_a_classic_response() { + let agent = stub::serve_once("400 Bad Request", r#"{"error":"Report data is too long"}"#); + let failure = dstack_attestation(&agent.socket, &ctx(), Some("n")) + .await + .unwrap_err(); + assert_eq!( + failure, + ( + Status::ServiceUnavailable, + "quote generation failed".to_string() + ) + ); + } + + #[tokio::test] + async fn quote_with_other_reportdata_is_a_503_even_when_the_echo_matches() { + let agent = stub::serve_quote_with(|_| hex::encode(stub::tdx_quote(&[0x24; 64]))); + let failure = dstack_attestation(&agent.socket, &ctx(), Some("n")) + .await + .unwrap_err(); + assert_eq!(failure.0, Status::ServiceUnavailable); + } + + #[tokio::test] + async fn truncated_quote_is_a_503() { + let agent = stub::serve_quote_with(|padded| hex::encode(&stub::tdx_quote(padded)[..600])); + let failure = dstack_attestation(&agent.socket, &ctx(), Some("n")) + .await + .unwrap_err(); + assert_eq!(failure.0, Status::ServiceUnavailable); + } + #[tokio::test] async fn quote_request_carries_a_32_byte_binding_hash() { let (_, sent) = attest(Some("a-verifier-chosen-nonce-of-some-length")).await; @@ -140,8 +199,9 @@ mod tests { #[tokio::test] async fn verifier_recomputes_report_data_from_the_returned_preimage() { let nonce = "verifier-nonce-1"; - let (response, sent) = attest(Some(nonce)).await; + let (response, _) = attest(Some(nonce)).await; let AttestationResponse::Dstack { + quote, enforcer_did, key_binding, key_binding_sha256, @@ -158,10 +218,77 @@ mod tests { ); let digest = sha2::Sha256::digest(key_binding.as_bytes()); assert_eq!(key_binding_sha256, hex::encode(digest)); - // dstack zero-pads report_data into the quote's 64-byte REPORTDATA. - let mut report_data = sent; - report_data.resize(crate::dstack::MAX_REPORT_DATA_BYTES, 0); + // Read REPORTDATA from the returned quote (TDX v4: header 48 bytes, + // REPORTDATA at TD report body offset 520): digest, then 32 zeros. + let quote = hex::decode(quote).unwrap(); + let report_data = "e[48 + 520..48 + 584]; assert_eq!(&report_data[..32], &digest[..]); assert_eq!(&report_data[32..], &[0u8; 32]); } + + /// A node with no TEE behind a limiter allowing `burst` requests per IP. + async fn client(burst: u32) -> rocket::local::asynchronous::Client { + let limits = crate::config::PublicSpacesConfig { + rate_limit_per_minute: 0, + rate_limit_burst: burst, + ..Default::default() + }; + let rocket = rocket::build() + .manage(None::) + .manage(crate::routes::public::RateLimiter::new(&limits)) + .mount("/", rocket::routes![attestation]); + rocket::local::asynchronous::Client::untracked(rocket) + .await + .unwrap() + } + + /// GET `uri` from client address `ip`; the limiter keys on it. + async fn status_from( + client: &rocket::local::asynchronous::Client, + ip: &str, + uri: &str, + ) -> Status { + let remote = format!("{ip}:4000").parse().unwrap(); + client + .get(uri.to_owned()) + .remote(remote) + .dispatch() + .await + .status() + } + + #[tokio::test] + async fn nonce_must_be_1_to_64_url_safe_characters() { + let client = client(10).await; + let longest = "n".repeat(64); + let too_long = "n".repeat(65); + for (nonce, status) in [ + ("abc-DEF_123", Status::Ok), + (longest.as_str(), Status::Ok), + (too_long.as_str(), Status::BadRequest), + ("a:b", Status::BadRequest), + ("", Status::BadRequest), + ] { + let got = + status_from(&client, "192.0.2.1", &format!("/attestation?nonce={nonce}")).await; + assert_eq!(got, status, "nonce {nonce:?}"); + } + } + + #[tokio::test] + async fn attestation_is_rate_limited_per_client() { + let client = client(1).await; + assert_eq!( + status_from(&client, "192.0.2.1", "/attestation").await, + Status::Ok + ); + assert_eq!( + status_from(&client, "192.0.2.1", "/attestation").await, + Status::TooManyRequests + ); + assert_eq!( + status_from(&client, "192.0.2.2", "/attestation").await, + Status::Ok + ); + } } From 879b90a4f4298142525b3a58c34cfd2e61cca22b Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 07:51:06 +0000 Subject: [PATCH 3/4] fix(attestation): keep guest-agent error bodies out of default logs (TC-963) --- docs/attestation.md | 17 ++++-- tinycloud-node-server/src/dstack.rs | 15 ++++-- .../src/routes/attestation.rs | 52 ++++++++++++++++++- 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/docs/attestation.md b/docs/attestation.md index 2ffdbef0..af1674eb 100644 --- a/docs/attestation.md +++ b/docs/attestation.md @@ -45,10 +45,13 @@ or TCB status, so a 200 response proves nothing on its own. with 200 and `mode: "classic"`. - A node outside a TEE returns 200 with `mode: "classic"`. A verifier that expects a TEE must treat that as a failure. +- The quote binds `enforcer_did` to the measured node. It does not prove that + the DID's private key was derived inside the TEE. ## Verifying -1. Choose a fresh, unpredictable nonce and request +1. Choose a fresh, unpredictable nonce, for example 32 random bytes as + unpadded base64url (43 characters), and request `/attestation?nonce=`. Reject anything but 200 with `mode: "dstack"`. 2. Build the expected preimage yourself from the DID you expect and your @@ -60,10 +63,16 @@ or TCB status, so a 200 response proves nothing on its own. including its TCB status, and read REPORTDATA from the verified quote, not from the response JSON. REPORTDATA must be the digest followed by 32 zero bytes. -5. Check the quote's MRTD and RTMR0-2 against the dstack OS image you trust. +5. Check the TD report body yourself; quote verification doesn't apply this + policy (Intel TDX DCAP Quoting Library API, ยง2.3.2 and Appendix A.3.4). + TDATTRIBUTES must have the debug bit (TUD) clear, since a debug TD's memory + is readable by the host. +6. Check the quote's MRTD and RTMR0-2 against the dstack OS image you trust. Replay `event_log` against RTMR3 and take `compose-hash` and `app-id` from - the replayed events, not from the JSON fields; compare them with the - release you expect. + the replayed events, not from the JSON fields. `compose-hash` is SHA-256 of + the full app-compose JSON dstack deployed (compose file plus runner + settings), so compare it with a trusted hash of that exact manifest for the + release you expect, and check the manifest pins the expected image digests. The digest only binds what the verifier recomputes in steps 2 and 3, so always derive the preimage from your own nonce and expected DID rather than diff --git a/tinycloud-node-server/src/dstack.rs b/tinycloud-node-server/src/dstack.rs index 6a135a0a..134da8b4 100644 --- a/tinycloud-node-server/src/dstack.rs +++ b/tinycloud-node-server/src/dstack.rs @@ -59,9 +59,18 @@ fn parse_response_body(raw: &[u8]) -> Result<&[u8]> { .ok_or_else(|| anyhow!("invalid HTTP response: missing status code"))?; let body = &raw[header_end + 4..]; if !(200..300).contains(&status) { - // The error body stays in the operator log; callers see only the status. - let excerpt = String::from_utf8_lossy(&body[..body.len().min(512)]); - tracing::warn!(status, body = %excerpt, "dstack guest agent returned an error"); + // Production logs ship off-host, so they get the status and size only; + // the agent's body is logged at debug, which is off by default. + tracing::warn!( + status, + body_len = body.len(), + "dstack guest agent returned an error" + ); + tracing::debug!( + status, + body = %String::from_utf8_lossy(&body[..body.len().min(512)]), + "dstack guest agent error body" + ); anyhow::bail!("guest agent returned HTTP {status}"); } Ok(body) diff --git a/tinycloud-node-server/src/routes/attestation.rs b/tinycloud-node-server/src/routes/attestation.rs index f9910adf..74d13145 100644 --- a/tinycloud-node-server/src/routes/attestation.rs +++ b/tinycloud-node-server/src/routes/attestation.rs @@ -94,7 +94,10 @@ async fn dstack_attestation( let quote_resp = crate::dstack::get_quote(socket, &report_data) .await .map_err(|e| { - tracing::error!("Failed to get TDX quote: {:#}", e); + // `{e}` is only the outermost context, a fixed message from this + // code; the full chain may quote agent output, so it is debug-only. + tracing::error!(error_class = %e, "TDX quote unavailable"); + tracing::debug!("TDX quote failure: {e:#}"); ( Status::ServiceUnavailable, "quote generation failed".to_string(), @@ -159,6 +162,53 @@ mod tests { ); } + /// Log lines written at production's default INFO level. + #[derive(Clone, Default)] + struct Captured(std::sync::Arc>>); + + impl std::io::Write for Captured { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Captured { + type Writer = Captured; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + #[tokio::test] + async fn quote_failure_logs_status_and_length_but_not_the_agent_body() { + let logs = Captured::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_max_level(tracing::Level::INFO) + .with_ansi(false) + .finish(); + // #[tokio::test] runs on one thread, so this covers the whole request. + let _guard = tracing::subscriber::set_default(subscriber); + let body = r#"{"error":"agent-detail-7f3a"}"#; + let agent = stub::serve_once("400 Bad Request", body); + + dstack_attestation(&agent.socket, &ctx(), Some("n")) + .await + .unwrap_err(); + + let logged = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + assert!(!logged.contains("agent-detail-7f3a"), "{logged}"); + assert!(logged.contains("status=400"), "{logged}"); + assert!( + logged.contains(&format!("body_len={}", body.len())), + "{logged}" + ); + } + #[tokio::test] async fn quote_with_other_reportdata_is_a_503_even_when_the_echo_matches() { let agent = stub::serve_quote_with(|_| hex::encode(stub::tdx_quote(&[0x24; 64]))); From eb07ae68d072c81a8cb23d0aa317ba43f785cf18 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 08:03:43 +0000 Subject: [PATCH 4/4] test(attestation): isolate log redaction test in its own binary (TC-963) --- tinycloud-node-server/src/lib.rs | 1 - .../src/routes/attestation.rs | 47 ----------- .../tests/attestation_logging.rs | 77 +++++++++++++++++++ 3 files changed, 77 insertions(+), 48 deletions(-) create mode 100644 tinycloud-node-server/tests/attestation_logging.rs diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 5e63f530..f2c49408 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -28,7 +28,6 @@ pub mod allow_list; pub mod auth_guards; pub mod authorization; pub mod config; -#[cfg(any(test, feature = "dstack"))] pub mod dstack; pub mod egress; pub mod hooks; diff --git a/tinycloud-node-server/src/routes/attestation.rs b/tinycloud-node-server/src/routes/attestation.rs index 74d13145..d7582a86 100644 --- a/tinycloud-node-server/src/routes/attestation.rs +++ b/tinycloud-node-server/src/routes/attestation.rs @@ -162,53 +162,6 @@ mod tests { ); } - /// Log lines written at production's default INFO level. - #[derive(Clone, Default)] - struct Captured(std::sync::Arc>>); - - impl std::io::Write for Captured { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - self.0.lock().unwrap().extend_from_slice(bytes); - Ok(bytes.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Captured { - type Writer = Captured; - fn make_writer(&'a self) -> Self::Writer { - self.clone() - } - } - - #[tokio::test] - async fn quote_failure_logs_status_and_length_but_not_the_agent_body() { - let logs = Captured::default(); - let subscriber = tracing_subscriber::fmt() - .with_writer(logs.clone()) - .with_max_level(tracing::Level::INFO) - .with_ansi(false) - .finish(); - // #[tokio::test] runs on one thread, so this covers the whole request. - let _guard = tracing::subscriber::set_default(subscriber); - let body = r#"{"error":"agent-detail-7f3a"}"#; - let agent = stub::serve_once("400 Bad Request", body); - - dstack_attestation(&agent.socket, &ctx(), Some("n")) - .await - .unwrap_err(); - - let logged = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); - assert!(!logged.contains("agent-detail-7f3a"), "{logged}"); - assert!(logged.contains("status=400"), "{logged}"); - assert!( - logged.contains(&format!("body_len={}", body.len())), - "{logged}" - ); - } - #[tokio::test] async fn quote_with_other_reportdata_is_a_503_even_when_the_echo_matches() { let agent = stub::serve_quote_with(|_| hex::encode(stub::tdx_quote(&[0x24; 64]))); diff --git a/tinycloud-node-server/tests/attestation_logging.rs b/tinycloud-node-server/tests/attestation_logging.rs new file mode 100644 index 00000000..8c5fcd28 --- /dev/null +++ b/tinycloud-node-server/tests/attestation_logging.rs @@ -0,0 +1,77 @@ +//! TC-963: when the dstack guest agent answers GetQuote with an error, the +//! node's logs at the production level carry its HTTP status and body length, +//! never the body. Production logs leave the host (TC-959, TC-961). +//! +//! This is its own test binary with one test, so the process-wide subscriber +//! sees every event; a thread-scoped subscriber inside the library's parallel +//! unit tests can miss events while other threads register the same callsites. + +use std::sync::{Arc, Mutex}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixListener; + +/// Log output written at production's default INFO level. +#[derive(Clone, Default)] +struct Captured(Arc>>); + +impl std::io::Write for Captured { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Captured { + type Writer = Captured; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } +} + +#[tokio::test] +async fn guest_agent_error_body_stays_out_of_production_logs() { + let logs = Captured::default(); + tracing::subscriber::set_global_default( + tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_max_level(tracing::Level::INFO) + .with_ansi(false) + .finish(), + ) + .unwrap(); + + let body = r#"{"error":"agent-detail-7f3a"}"#; + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("dstack.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let agent = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + stream.read_to_end(&mut request).await.unwrap(); + let response = format!( + "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + stream.shutdown().await.unwrap(); + }); + + let err = tinycloud::dstack::get_quote(socket.to_str().unwrap(), &[0x42; 32]) + .await + .unwrap_err(); + agent.await.unwrap(); + + assert!(format!("{err:#}").contains("HTTP 400"), "{err:#}"); + let logged = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + assert!(!logged.contains("agent-detail-7f3a"), "{logged}"); + assert!(logged.contains("status=400"), "{logged}"); + assert!( + logged.contains(&format!("body_len={}", body.len())), + "{logged}" + ); +}