From 94a59ff4dbd35f43b66541b80a40cfc37b81eda8 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 03:43:01 +0000 Subject: [PATCH 1/2] TC-1012 cap /invoke upload bodies at 95 MiB with 413 --- .github/workflows/docker.yml | 2 +- CHANGELOG.md | 1 + deploy/ingress/test-redacted-logging.sh | 15 ++- docs/tc-910-ingress-header-buffers.md | 4 +- tinycloud-node-server/src/routes/mod.rs | 154 +++++++++++++++++++++--- 5 files changed, 152 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7bdc0b1f..b50130b3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -350,7 +350,7 @@ jobs: - uses: actions/checkout@v4 # TC-970: no ticket, identifier or client address in the ingress logs; - # TC-910 header buffers and the 32 MB body limit still hold. Runs before + # TC-910 header buffers and the 95 MiB body limit still hold. Runs before # the push so a failing wrapper is never published. - name: Test redacted ingress logging run: deploy/ingress/test-redacted-logging.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index ffc71dd8..613d759d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Tighten revocation scope checks (TC-1302). - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). +- `/invoke` upload bodies are capped at 95 MiB (99,614,720 bytes), and a longer body gets 413 (TC-1012). This applies to a KV put, a multipart KV batch put (the whole multipart body) and a DuckDB import, whether the body is declared or chunked. Before, a KV put kept only the first 1 GB of a longer body without an error, a batch put over 1 GB got 400, and a DuckDB import was cut off at 100 MB. The ingress test now expects `CLIENT_MAX_BODY_SIZE=95m`. The release compose line changes with the TC-959 N0 compose (#309). ## [1.20.7] - 2026-10-14 diff --git a/deploy/ingress/test-redacted-logging.sh b/deploy/ingress/test-redacted-logging.sh index 0e8411f4..61c9d1ad 100755 --- a/deploy/ingress/test-redacted-logging.sh +++ b/deploy/ingress/test-redacted-logging.sh @@ -4,7 +4,7 @@ # credentials and identifiers from a separate client container, and check that # none of them, nor the client address, reaches the ingress container's # stdout/stderr or any file under /var/log/nginx. Also re-checks the TC-910 -# 32 KB header buffers and the production compose's body limit (32 MB, TC-940). +# 32 KB header buffers and the production compose's body limit (95 MiB, TC-1012). # # Usage: deploy/ingress/test-redacted-logging.sh # IMAGE= test an existing image instead of building deploy/ingress @@ -87,7 +87,7 @@ exec nginx -g "daemon off;" $DOCKER network create "$NETWORK" >/dev/null $DOCKER run -d --name "$INGRESS" --network "$NETWORK" \ -e DOMAIN="$DOMAIN" -e PORT=443 -e TARGET_ENDPOINT=http://127.0.0.1:8000 \ - -e CLIENT_MAX_BODY_SIZE=32m \ + -e CLIENT_MAX_BODY_SIZE=95m \ --entrypoint /bin/bash "$IMAGE" -c "$INGRESS_SETUP" >/dev/null $DOCKER run -d --name "$CLIENT" --network "$NETWORK" --entrypoint sleep "$IMAGE" infinity >/dev/null @@ -141,10 +141,13 @@ expect_status 400 --http1.1 -H "X-Large: $(head -c 40000 /dev/zero | tr '\0' a)" LARGE_HEADER="X-Large: $(head -c 30000 /dev/zero | tr '\0' a)" expect_status 200 -H "$LARGE_HEADER" "https://$DOMAIN/delegate" -# TC-940: CLIENT_MAX_BODY_SIZE=32m admits 30 MB and refuses more than 32 MiB. -$DOCKER exec "$CLIENT" sh -c 'head -c 30000000 /dev/zero >/tmp/under && head -c 34000000 /dev/zero >/tmp/over' -expect_status 200 -X POST --data-binary @/tmp/under "https://$DOMAIN/invoke" +# TC-1012: CLIENT_MAX_BODY_SIZE=95m admits exactly 95 MiB (99,614,720 bytes) +# and refuses one byte more, declared or chunked. +$DOCKER exec "$CLIENT" sh -c 'head -c 99614720 /dev/zero >/tmp/at && head -c 99614721 /dev/zero >/tmp/over' +expect_status 200 -X POST --data-binary @/tmp/at "https://$DOMAIN/invoke" expect_status 413 -X POST --data-binary @/tmp/over "https://$DOMAIN/invoke" +expect_status 413 --http1.1 -H "Transfer-Encoding: chunked" -X POST --data-binary @/tmp/over \ + "https://$DOMAIN/invoke" sleep 1 LOGS=$($DOCKER logs "$INGRESS" 2>&1) @@ -207,4 +210,4 @@ expect_line POST "/invoke" 413 echo "ingress container output:" sed 's/^/ /' <<<"$LOGS" -echo "PASS: no ticket, identifier or client address in the ingress logs; 32 KB headers and 32 MB bodies still accepted" +echo "PASS: no ticket, identifier or client address in the ingress logs; 32 KB headers and 95 MiB bodies accepted, larger bodies refused" diff --git a/docs/tc-910-ingress-header-buffers.md b/docs/tc-910-ingress-header-buffers.md index 0b0279ed..a182fa58 100644 --- a/docs/tc-910-ingress-header-buffers.md +++ b/docs/tc-910-ingress-header-buffers.md @@ -2,7 +2,7 @@ The tee.node production ingress has nginx's default `large_client_header_buffers 4 8k`. Session activation (`POST /delegate`) places the signed delegation in one `Authorization` header; replication login headers can exceed 8 KiB and fail with `400 Request Header Or Cookie Too Large`. -`deploy/ingress/Dockerfile` wraps the pinned `dstacktee/dstack-ingress:1.4` image and copies `header-buffers.conf` to `/etc/nginx/conf.d/00-header-buffers.conf`. 1.20.1 shipped the wrapper on `dstack-ingress:20250924`; #261 moved it to `1.4`, which also honours `CLIENT_MAX_BODY_SIZE`. The compose sets it to 32 MB and leaves the proxy timeouts at nginx's defaults (TC-940). The directive `large_client_header_buffers 4 32k;` resides in the nginx `http {}` context. The prefixed filename is loaded before its generated `default.conf` server block. This raises the per-header-line capacity while retaining four buffers. +`deploy/ingress/Dockerfile` wraps the pinned `dstacktee/dstack-ingress:1.4` image and copies `header-buffers.conf` to `/etc/nginx/conf.d/00-header-buffers.conf`. 1.20.1 shipped the wrapper on `dstack-ingress:20250924`; #261 moved it to `1.4`, which also honours `CLIENT_MAX_BODY_SIZE`. The compose sets it to 95 MiB, matching the node's own upload limit (TC-1012; 32 MB under TC-940), and leaves the proxy timeouts at nginx's defaults. The directive `large_client_header_buffers 4 32k;` resides in the nginx `http {}` context. The prefixed filename is loaded before its generated `default.conf` server block. This raises the per-header-line capacity while retaining four buffers. The upstream entrypoint's `setup_nginx_conf` generates `default.conf`; it does not remove the custom `00-header-buffers.conf` drop-in. The directive is valid in `http` context. @@ -23,7 +23,7 @@ Size: the access log goes to the container's stdout. The compose caps the ingres `deploy/ingress/test-redacted-logging.sh` builds the wrapper and runs it with the server block generated by the entrypoint's own `setup_nginx_conf`, using a self-signed certificate. It sends credential-bearing requests from a second container and asserts: - none of the markers or the client address appear in stdout, stderr or `/var/log/nginx`; - each request is logged as a valid JSON line with the expected method, route class and status, over HTTP/2 and HTTP/1.1, including a 400 raised before routing; -- a 30 KB header and a 30 MB body are accepted, and a body over 32 MiB is refused with 413 (the compose sets `CLIENT_MAX_BODY_SIZE=32m`). +- a 30 KB header and a body of exactly 95 MiB (99,614,720 bytes) are accepted, and a body one byte larger is refused with 413, declared or chunked (the compose sets `CLIENT_MAX_BODY_SIZE=95m`). CI runs it in the `build-ingress` job before the image is pushed. Locally: `DOCKER="sudo docker" deploy/ingress/test-redacted-logging.sh`, or `IMAGE=` to test a published wrapper. diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index 43031cb5..4c6f3c53 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -79,6 +79,30 @@ use util::LimitedReader; const MAX_KV_BATCH_READ_ITEMS: usize = 100; const UPLOAD_COPY_BUFFER_SIZE: usize = 1024 * 1024; +/// Largest `/invoke` upload body, in bytes: 95 MiB (TC-1012). It matches the +/// ingress `client_max_body_size` and the tinycloud-tee-proxy Worker cap; a +/// longer body is refused with 413. +const MAX_UPLOAD_BODY_BYTES: u64 = 95 * 1024 * 1024; + +fn upload_body_too_large() -> (Status, String) { + ( + Status::PayloadTooLarge, + format!("Request body exceeds the {MAX_UPLOAD_BODY_BYTES}-byte upload limit"), + ) +} + +/// Opens an upload body one byte past [`MAX_UPLOAD_BODY_BYTES`], so its +/// reader can tell a body of exactly the limit from a longer one. +fn open_upload_body(data: rocket::Data<'_>) -> rocket::data::DataStream<'_> { + data.open((MAX_UPLOAD_BODY_BYTES + 1).bytes()) +} + +fn multipart_error(error: multer::Error) -> (Status, String) { + match error { + multer::Error::StreamSizeExceeded { .. } => upload_body_too_large(), + error => (Status::BadRequest, error.to_string()), + } +} async fn copy_upload(reader: R, writer: &mut W) -> std::io::Result where @@ -1460,11 +1484,7 @@ async fn copy_multipart_field_to_stage( space: &SpaceId, remaining: &mut Option<(u64, StorageCheck)>, ) -> Result<(), InvokeError> { - while let Some(chunk) = field - .chunk() - .await - .map_err(|e| (Status::BadRequest, e.to_string()))? - { + while let Some(chunk) = field.chunk().await.map_err(multipart_error)? { if let Some((remaining_bytes, check)) = remaining.as_mut() { let chunk_len = u64::try_from(chunk.len()) .map_err(|e| (Status::InternalServerError, e.to_string()))?; @@ -1506,7 +1526,13 @@ async fn build_batch_kv_inputs( })?; let boundary = multer::parse_boundary(content_type).map_err(|e| (Status::BadRequest, e.to_string()))?; - let mut multipart = multer::Multipart::with_reader(data.open(1u8.gigabytes()), boundary); + let constraints = multer::Constraints::new() + .size_limit(multer::SizeLimit::new().whole_stream(MAX_UPLOAD_BODY_BYTES)); + let mut multipart = multer::Multipart::with_reader_with_constraints( + open_upload_body(data), + boundary, + constraints, + ); let mut inputs = HashMap::new(); let (space, _) = expected .values() @@ -1515,11 +1541,7 @@ async fn build_batch_kv_inputs( let mut remaining = staged_batch_remaining(space, tinycloud, config, quota_cache).await?; let decode_start = Instant::now(); let result: Result = async { - while let Some(field) = multipart - .next_field() - .await - .map_err(|e| (Status::BadRequest, e.to_string()))? - { + while let Some(field) = multipart.next_field().await.map_err(multipart_error)? { let encoded_path = field .name() .ok_or_else(|| { @@ -1933,7 +1955,7 @@ async fn invoke_impl( .stage(space) .await .map_err(|e| (Status::InternalServerError, e.to_string()))?; - let open_data = d.open(1u8.gigabytes()).compat(); + let open_data = open_upload_body(d).compat(); // Public spaces use their storage cap, others the per-space quota. let budget = staged_batch_remaining(space, tinycloud, config, quota_cache).await?; @@ -1967,7 +1989,9 @@ async fn invoke_impl( ); result }; - copy_result?; + if copy_result? > MAX_UPLOAD_BODY_BYTES { + return Err(upload_body_too_large().into()); + } let mut inputs = HashMap::new(); inputs.insert( @@ -3114,17 +3138,20 @@ async fn handle_duckdb_invoke( if ability == "tinycloud.duckdb/import" { // Import always grows the database artifact — gate before reading - // the (up to 100 MB) body. Same 402 semantics as the KV/SQL paths. + // the (up to 95 MiB) body. Same 402 semantics as the KV/SQL paths. staged_batch_remaining(space, tinycloud, config, quota_cache).await?; let body_bytes = match data { DataIn::One(d) => { let mut buf = Vec::new(); - let mut reader = d.open(100u8.megabytes()); + let mut reader = open_upload_body(d); reader .read_to_end(&mut buf) .await .map_err(|e| (Status::BadRequest, e.to_string()))?; + if buf.len() as u64 > MAX_UPLOAD_BODY_BYTES { + return Err(upload_body_too_large().into()); + } buf } _ => { @@ -8354,6 +8381,103 @@ mod tests { Ok(()) } + /// Dispatch one signed `kv/put` of `doc` with `body` and `content_type` + /// through the real `/invoke` route; returns the status and body text. + async fn tc1012_kv_put( + nonce: &str, + content_type: rocket::http::ContentType, + body: Vec, + ) -> Result<(rocket::http::Status, String)> { + use rocket::http::Header; + use rocket::local::asynchronous::Client; + + let signer = JWK::generate_ed25519()?; + let issuer_vm = tc341_verification_method(&signer)?; + let (header, space) = tc409_root_invocation( + &signer, + &issuer_vm, + "kv", + "tinycloud.kv/put", + Some("doc"), + nonce, + )?; + let (rocket, _conn) = tc409_invoke_rocket(&space).await?; + let client = Client::tracked(rocket).await?; + let response = client + .post("/invoke") + .header(Header::new("Authorization", header)) + .header(content_type) + .body(body) + .dispatch() + .await; + let status = response.status(); + Ok((status, response.into_string().await.unwrap_or_default())) + } + + #[tokio::test] + async fn tc1012_kv_put_accepts_exactly_95_mib_and_refuses_one_more_byte_with_413() -> Result<()> + { + assert_eq!(MAX_UPLOAD_BODY_BYTES, 99_614_720); + let cap = MAX_UPLOAD_BODY_BYTES as usize; + + let (status, body) = tc1012_kv_put( + "urn:uuid:tc1012-kv-put-at-cap", + rocket::http::ContentType::Binary, + vec![0x5a; cap], + ) + .await?; + assert!(status.class().is_success(), "{status}: {body}"); + + let (status, body) = tc1012_kv_put( + "urn:uuid:tc1012-kv-put-over-cap", + rocket::http::ContentType::Binary, + vec![0x5a; cap + 1], + ) + .await?; + assert_eq!(status, rocket::http::Status::PayloadTooLarge, "{body}"); + Ok(()) + } + + #[tokio::test] + async fn tc1012_multipart_kv_put_accepts_95_mib_body_and_refuses_one_more_byte_with_413( + ) -> Result<()> { + let boundary = "tc1012-boundary"; + let multipart_body = |total: usize| { + let head = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"doc\"\r\n\ + Content-Type: application/octet-stream\r\n\r\n" + ); + let tail = format!("\r\n--{boundary}--\r\n"); + let mut body = head.into_bytes(); + body.resize(total - tail.len(), 0x5a); + body.extend_from_slice(tail.as_bytes()); + assert_eq!(body.len(), total); + body + }; + let content_type = || { + rocket::http::ContentType::new("multipart", "form-data") + .with_params(("boundary", boundary)) + }; + let cap = MAX_UPLOAD_BODY_BYTES as usize; + + let (status, body) = tc1012_kv_put( + "urn:uuid:tc1012-multipart-at-cap", + content_type(), + multipart_body(cap), + ) + .await?; + assert!(status.class().is_success(), "{status}: {body}"); + + let (status, body) = tc1012_kv_put( + "urn:uuid:tc1012-multipart-over-cap", + content_type(), + multipart_body(cap + 1), + ) + .await?; + assert_eq!(status, rocket::http::Status::PayloadTooLarge, "{body}"); + Ok(()) + } + #[tokio::test] async fn tc409_sql_shared_auth_verifies_signature_exactly_once() -> Result<()> { use rocket::http::{ContentType, Header}; From 494a832e856e9151b30512ea17be5d788a39bee2 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sat, 17 Oct 2026 03:55:06 +0000 Subject: [PATCH 2/2] TC-1012 CHANGELOG: release compose line rides with #315 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 613d759d..df3738b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ - Tighten revocation scope checks (TC-1302). - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195). - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963). -- `/invoke` upload bodies are capped at 95 MiB (99,614,720 bytes), and a longer body gets 413 (TC-1012). This applies to a KV put, a multipart KV batch put (the whole multipart body) and a DuckDB import, whether the body is declared or chunked. Before, a KV put kept only the first 1 GB of a longer body without an error, a batch put over 1 GB got 400, and a DuckDB import was cut off at 100 MB. The ingress test now expects `CLIENT_MAX_BODY_SIZE=95m`. The release compose line changes with the TC-959 N0 compose (#309). +- `/invoke` upload bodies are capped at 95 MiB (99,614,720 bytes), and a longer body gets 413 (TC-1012). This applies to a KV put, a multipart KV batch put (the whole multipart body) and a DuckDB import, whether the body is declared or chunked. Before, a KV put kept only the first 1 GB of a longer body without an error, a batch put over 1 GB got 400, and a DuckDB import was cut off at 100 MB. The ingress test now expects `CLIENT_MAX_BODY_SIZE=95m`. The release compose line changes with the TC-959 N0 compose (#315). ## [1.20.7] - 2026-10-14