TC-1012: cap /invoke upload bodies at 95 MiB with 413

.github/workflows/docker.yml

@@ -416,7 +416,7 @@ jobs:
       - uses: actions/checkout@v4
 
       # TC-970: no ticket, identifier or client address in the ingress logs;
-      # TC-910 header buffers and the 32 MB body limit still hold. Runs before
+      # TC-910 header buffers and the 95 MiB body limit still hold. Runs before
       # the push so a failing wrapper is never published.
       - name: Test redacted ingress logging
         run: deploy/ingress/test-redacted-logging.sh

CHANGELOG.md

@@ -20,6 +20,7 @@
 - Account delegation session discovery keyset-pages delegation rows independently of ability fan-out, using a composite delegator/id index so large grant sets are not truncated (TC-1195).
 - On PostgreSQL, the boot's schema migrations wait at most 30 s for any lock (`SET LOCAL lock_timeout`, in the same transaction that lifts `statement_timeout`) (TC-1176). If another session holds a conflicting lock on a table a migration changes, such as an open write on `delegation` during the first boot after 1.20.6, the node now fails to start with an error naming the lock wait instead of hanging. All pending migrations roll back together, so nothing is applied or recorded, and the next start applies them. The limit ends with that transaction and does not reach pooled sessions. SQLite and MySQL are unchanged.
 - `/attestation` quotes the SHA-256 of `key_binding` as REPORTDATA and returns it as `key_binding_sha256`; the quote no longer carries the raw binding. Before answering 200 the node checks that the returned TDX v4 quote's REPORTDATA field holds that digest (a structural check; it does not verify the quote). When the TEE can't produce a bound quote, the endpoint returns 503 instead of 200 `mode: "classic"`, and the dstack client reports a guest-agent error status instead of a parse error. `nonce` must be 1-64 characters of `A-Z a-z 0-9 - _` (400 otherwise), and the endpoint now shares the public routes' per-IP rate limit. Verifiers recompute the digest from their own nonce and expected DID and must still verify the quote; see `docs/attestation.md` (TC-963).
+- `/invoke` upload bodies are capped at 95 MiB (99,614,720 bytes), and a longer body gets 413 (TC-1012). This applies to a KV put, a multipart KV batch put (the whole multipart body) and a DuckDB import, whether the body is declared or chunked. Before, a KV put kept only the first 1 GB of a longer body without an error, a batch put over 1 GB got 400, and a DuckDB import was cut off at 100 MB. The ingress test now expects `CLIENT_MAX_BODY_SIZE=95m`. The release compose line changes with the TC-959 N0 compose (#315).
 - Prepare a compose-only GELF-to-Vector sidecar for the unchanged node 1.20.6 and ingress image digests. The default unsampled `errors_slow` profile emits valid requests with status >=400 or duration strictly over one second; explicit `all_requests` remains available, and all accepted typed WARN/ERROR diagnostics remain eligible for bounded count aggregation. Document eligible-request volume accounting, accepted BEST-EFFORT outage loss, the failed local burst evidence and required zero-observed-loss healthy hosted burst, plus the config-only deployment/rollback privacy packet. No Rust or application image change and no deployment authorization; see `docs/runbooks/tc-959-n0-log-shipping.md` (TC-959 N0).
 
 ## [1.20.7] - 2026-10-14

deploy/ingress/test-redacted-logging.sh

@@ -4,7 +4,7 @@
 # credentials and identifiers from a separate client container, and check that
 # none of them, nor the client address, reaches the ingress container's
 # stdout/stderr or any file under /var/log/nginx. Also re-checks the TC-910
-# 32 KB header buffers and the production compose's body limit (32 MB, TC-940).
+# 32 KB header buffers and the production compose's body limit (95 MiB, TC-1012).
 #
 # Usage: deploy/ingress/test-redacted-logging.sh
 #   IMAGE=<ref>   test an existing image instead of building deploy/ingress
@@ -87,7 +87,7 @@ exec nginx -g "daemon off;"
 $DOCKER network create "$NETWORK" >/dev/null
 $DOCKER run -d --name "$INGRESS" --network "$NETWORK" \
     -e DOMAIN="$DOMAIN" -e PORT=443 -e TARGET_ENDPOINT=http://127.0.0.1:8000 \
-    -e CLIENT_MAX_BODY_SIZE=32m \
+    -e CLIENT_MAX_BODY_SIZE=95m \
     --entrypoint /bin/bash "$IMAGE" -c "$INGRESS_SETUP" >/dev/null
 $DOCKER run -d --name "$CLIENT" --network "$NETWORK" --entrypoint sleep "$IMAGE" infinity >/dev/null
 
@@ -141,10 +141,13 @@ expect_status 400 --http1.1 -H "X-Large: $(head -c 40000 /dev/zero | tr '\0' a)"
 LARGE_HEADER="X-Large: $(head -c 30000 /dev/zero | tr '\0' a)"
 expect_status 200 -H "$LARGE_HEADER" "https://$DOMAIN/delegate"
 
-# TC-940: CLIENT_MAX_BODY_SIZE=32m admits 30 MB and refuses more than 32 MiB.
-$DOCKER exec "$CLIENT" sh -c 'head -c 30000000 /dev/zero >/tmp/under && head -c 34000000 /dev/zero >/tmp/over'
-expect_status 200 -X POST --data-binary @/tmp/under "https://$DOMAIN/invoke"
+# TC-1012: CLIENT_MAX_BODY_SIZE=95m admits exactly 95 MiB (99,614,720 bytes)
+# and refuses one byte more, declared or chunked.
+$DOCKER exec "$CLIENT" sh -c 'head -c 99614720 /dev/zero >/tmp/at && head -c 99614721 /dev/zero >/tmp/over'
+expect_status 200 -X POST --data-binary @/tmp/at "https://$DOMAIN/invoke"
 expect_status 413 -X POST --data-binary @/tmp/over "https://$DOMAIN/invoke"
+expect_status 413 --http1.1 -H "Transfer-Encoding: chunked" -X POST --data-binary @/tmp/over \
+    "https://$DOMAIN/invoke"
 
 sleep 1
 LOGS=$($DOCKER logs "$INGRESS" 2>&1)
@@ -207,4 +210,4 @@ expect_line POST "/invoke" 413
 
 echo "ingress container output:"
 sed 's/^/  /' <<<"$LOGS"
-echo "PASS: no ticket, identifier or client address in the ingress logs; 32 KB headers and 32 MB bodies still accepted"
+echo "PASS: no ticket, identifier or client address in the ingress logs; 32 KB headers and 95 MiB bodies accepted, larger bodies refused"

docs/tc-910-ingress-header-buffers.md

@@ -2,7 +2,7 @@
 
 The tee.node production ingress has nginx's default `large_client_header_buffers 4 8k`. Session activation (`POST /delegate`) places the signed delegation in one `Authorization` header; replication login headers can exceed 8 KiB and fail with `400 Request Header Or Cookie Too Large`.
 
-`deploy/ingress/Dockerfile` wraps the pinned `dstacktee/dstack-ingress:1.4` image and copies `header-buffers.conf` to `/etc/nginx/conf.d/00-header-buffers.conf`. 1.20.1 shipped the wrapper on `dstack-ingress:20250924`; #261 moved it to `1.4`, which also honours `CLIENT_MAX_BODY_SIZE`. The compose sets it to 32 MB and leaves the proxy timeouts at nginx's defaults (TC-940). The directive `large_client_header_buffers 4 32k;` resides in the nginx `http {}` context. The prefixed filename is loaded before its generated `default.conf` server block. This raises the per-header-line capacity while retaining four buffers.
+`deploy/ingress/Dockerfile` wraps the pinned `dstacktee/dstack-ingress:1.4` image and copies `header-buffers.conf` to `/etc/nginx/conf.d/00-header-buffers.conf`. 1.20.1 shipped the wrapper on `dstack-ingress:20250924`; #261 moved it to `1.4`, which also honours `CLIENT_MAX_BODY_SIZE`. The compose sets it to 95 MiB, matching the node's own upload limit (TC-1012; 32 MB under TC-940), and leaves the proxy timeouts at nginx's defaults. The directive `large_client_header_buffers 4 32k;` resides in the nginx `http {}` context. The prefixed filename is loaded before its generated `default.conf` server block. This raises the per-header-line capacity while retaining four buffers.
 
 The upstream entrypoint's `setup_nginx_conf` generates `default.conf`; it does not remove the custom `00-header-buffers.conf` drop-in. The directive is valid in `http` context.
 
@@ -23,7 +23,7 @@ Size: the access log goes to the container's stdout. The compose caps the ingres
 `deploy/ingress/test-redacted-logging.sh` builds the wrapper and runs it with the server block generated by the entrypoint's own `setup_nginx_conf`, using a self-signed certificate. It sends credential-bearing requests from a second container and asserts:
 - none of the markers or the client address appear in stdout, stderr or `/var/log/nginx`;
 - each request is logged as a valid JSON line with the expected method, route class and status, over HTTP/2 and HTTP/1.1, including a 400 raised before routing;
-- a 30 KB header and a 30 MB body are accepted, and a body over 32 MiB is refused with 413 (the compose sets `CLIENT_MAX_BODY_SIZE=32m`).
+- a 30 KB header and a body of exactly 95 MiB (99,614,720 bytes) are accepted, and a body one byte larger is refused with 413, declared or chunked (the compose sets `CLIENT_MAX_BODY_SIZE=95m`).
 
 CI runs it in the `build-ingress` job before the image is pushed. Locally: `DOCKER="sudo docker" deploy/ingress/test-redacted-logging.sh`, or `IMAGE=<ref>` to test a published wrapper.
 

tinycloud-node-server/src/routes/mod.rs

@@ -79,6 +79,30 @@ use util::LimitedReader;
 
 const MAX_KV_BATCH_READ_ITEMS: usize = 100;
 const UPLOAD_COPY_BUFFER_SIZE: usize = 1024 * 1024;
+/// Largest `/invoke` upload body, in bytes: 95 MiB (TC-1012). It matches the
+/// ingress `client_max_body_size` and the tinycloud-tee-proxy Worker cap; a
+/// longer body is refused with 413.
+const MAX_UPLOAD_BODY_BYTES: u64 = 95 * 1024 * 1024;
+
+fn upload_body_too_large() -> (Status, String) {
+    (
+        Status::PayloadTooLarge,
+        format!("Request body exceeds the {MAX_UPLOAD_BODY_BYTES}-byte upload limit"),
+    )
+}
+
+/// Opens an upload body one byte past [`MAX_UPLOAD_BODY_BYTES`], so its
+/// reader can tell a body of exactly the limit from a longer one.
+fn open_upload_body(data: rocket::Data<'_>) -> rocket::data::DataStream<'_> {
+    data.open((MAX_UPLOAD_BODY_BYTES + 1).bytes())
+}
+
+fn multipart_error(error: multer::Error) -> (Status, String) {
+    match error {
+        multer::Error::StreamSizeExceeded { .. } => upload_body_too_large(),
+        error => (Status::BadRequest, error.to_string()),
+    }
+}
 
 async fn copy_upload<R, W>(reader: R, writer: &mut W) -> std::io::Result<u64>
 where
@@ -1458,11 +1482,7 @@ async fn copy_multipart_field_to_stage(
     space: &SpaceId,
     remaining: &mut Option<(u64, StorageCheck)>,
 ) -> Result<(), InvokeError> {
-    while let Some(chunk) = field
-        .chunk()
-        .await
-        .map_err(|e| (Status::BadRequest, e.to_string()))?
-    {
+    while let Some(chunk) = field.chunk().await.map_err(multipart_error)? {
         if let Some((remaining_bytes, check)) = remaining.as_mut() {
             let chunk_len = u64::try_from(chunk.len())
                 .map_err(|e| (Status::InternalServerError, e.to_string()))?;
@@ -1504,7 +1524,13 @@ async fn build_batch_kv_inputs(
     })?;
     let boundary =
         multer::parse_boundary(content_type).map_err(|e| (Status::BadRequest, e.to_string()))?;
-    let mut multipart = multer::Multipart::with_reader(data.open(1u8.gigabytes()), boundary);
+    let constraints = multer::Constraints::new()
+        .size_limit(multer::SizeLimit::new().whole_stream(MAX_UPLOAD_BODY_BYTES));
+    let mut multipart = multer::Multipart::with_reader_with_constraints(
+        open_upload_body(data),
+        boundary,
+        constraints,
+    );
     let mut inputs = HashMap::new();
     let (space, _) = expected
         .values()
@@ -1513,11 +1539,7 @@ async fn build_batch_kv_inputs(
     let mut remaining = staged_batch_remaining(space, tinycloud, config, quota_cache).await?;
     let decode_start = Instant::now();
     let result: Result<KvInputMap, InvokeError> = async {
-        while let Some(field) = multipart
-            .next_field()
-            .await
-            .map_err(|e| (Status::BadRequest, e.to_string()))?
-        {
+        while let Some(field) = multipart.next_field().await.map_err(multipart_error)? {
             let encoded_path = field
                 .name()
                 .ok_or_else(|| {
@@ -1934,7 +1956,7 @@ async fn invoke_impl(
                         .stage(space)
                         .await
                         .map_err(|e| (Status::InternalServerError, e.to_string()))?;
-                    let open_data = d.open(1u8.gigabytes()).compat();
+                    let open_data = open_upload_body(d).compat();
 
                     // Public spaces use their storage cap, others the per-space quota.
                     let budget = staged_batch_remaining(space, tinycloud, config, quota_cache).await?;
@@ -1968,7 +1990,9 @@ async fn invoke_impl(
                         );
                         result
                     };
-                    copy_result?;
+                    if copy_result? > MAX_UPLOAD_BODY_BYTES {
+                        return Err(upload_body_too_large().into());
+                    }
 
                     let mut inputs = HashMap::new();
                     inputs.insert(
@@ -3127,17 +3151,20 @@ async fn handle_duckdb_invoke(
 
     if ability == "tinycloud.duckdb/import" {
         // Import always grows the database artifact — gate before reading
-        // the (up to 100 MB) body. Same 402 semantics as the KV/SQL paths.
+        // the (up to 95 MiB) body. Same 402 semantics as the KV/SQL paths.
         staged_batch_remaining(space, tinycloud, config, quota_cache).await?;
 
         let body_bytes = match data {
             DataIn::One(d) => {
                 let mut buf = Vec::new();
-                let mut reader = d.open(100u8.megabytes());
+                let mut reader = open_upload_body(d);
                 reader
                     .read_to_end(&mut buf)
                     .await
                     .map_err(|e| (Status::BadRequest, e.to_string()))?;
+                if buf.len() as u64 > MAX_UPLOAD_BODY_BYTES {
+                    return Err(upload_body_too_large().into());
+                }
                 buf
             }
             _ => {
@@ -8371,6 +8398,103 @@ mod tests {
         Ok(())
     }
 
+    /// Dispatch one signed `kv/put` of `doc` with `body` and `content_type`
+    /// through the real `/invoke` route; returns the status and body text.
+    async fn tc1012_kv_put(
+        nonce: &str,
+        content_type: rocket::http::ContentType,
+        body: Vec<u8>,
+    ) -> Result<(rocket::http::Status, String)> {
+        use rocket::http::Header;
+        use rocket::local::asynchronous::Client;
+
+        let signer = JWK::generate_ed25519()?;
+        let issuer_vm = tc341_verification_method(&signer)?;
+        let (header, space) = tc409_root_invocation(
+            &signer,
+            &issuer_vm,
+            "kv",
+            "tinycloud.kv/put",
+            Some("doc"),
+            nonce,
+        )?;
+        let (rocket, _conn) = tc409_invoke_rocket(&space).await?;
+        let client = Client::tracked(rocket).await?;
+        let response = client
+            .post("/invoke")
+            .header(Header::new("Authorization", header))
+            .header(content_type)
+            .body(body)
+            .dispatch()
+            .await;
+        let status = response.status();
+        Ok((status, response.into_string().await.unwrap_or_default()))
+    }
+
+    #[tokio::test]
+    async fn tc1012_kv_put_accepts_exactly_95_mib_and_refuses_one_more_byte_with_413() -> Result<()>
+    {
+        assert_eq!(MAX_UPLOAD_BODY_BYTES, 99_614_720);
+        let cap = MAX_UPLOAD_BODY_BYTES as usize;
+
+        let (status, body) = tc1012_kv_put(
+            "urn:uuid:tc1012-kv-put-at-cap",
+            rocket::http::ContentType::Binary,
+            vec![0x5a; cap],
+        )
+        .await?;
+        assert!(status.class().is_success(), "{status}: {body}");
+
+        let (status, body) = tc1012_kv_put(
+            "urn:uuid:tc1012-kv-put-over-cap",
+            rocket::http::ContentType::Binary,
+            vec![0x5a; cap + 1],
+        )
+        .await?;
+        assert_eq!(status, rocket::http::Status::PayloadTooLarge, "{body}");
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn tc1012_multipart_kv_put_accepts_95_mib_body_and_refuses_one_more_byte_with_413(
+    ) -> Result<()> {
+        let boundary = "tc1012-boundary";
+        let multipart_body = |total: usize| {
+            let head = format!(
+                "--{boundary}\r\nContent-Disposition: form-data; name=\"doc\"\r\n\
+                 Content-Type: application/octet-stream\r\n\r\n"
+            );
+            let tail = format!("\r\n--{boundary}--\r\n");
+            let mut body = head.into_bytes();
+            body.resize(total - tail.len(), 0x5a);
+            body.extend_from_slice(tail.as_bytes());
+            assert_eq!(body.len(), total);
+            body
+        };
+        let content_type = || {
+            rocket::http::ContentType::new("multipart", "form-data")
+                .with_params(("boundary", boundary))
+        };
+        let cap = MAX_UPLOAD_BODY_BYTES as usize;
+
+        let (status, body) = tc1012_kv_put(
+            "urn:uuid:tc1012-multipart-at-cap",
+            content_type(),
+            multipart_body(cap),
+        )
+        .await?;
+        assert!(status.class().is_success(), "{status}: {body}");
+
+        let (status, body) = tc1012_kv_put(
+            "urn:uuid:tc1012-multipart-over-cap",
+            content_type(),
+            multipart_body(cap + 1),
+        )
+        .await?;
+        assert_eq!(status, rocket::http::Status::PayloadTooLarge, "{body}");
+        Ok(())
+    }
+
     #[tokio::test]
     async fn tc409_sql_shared_auth_verifies_signature_exactly_once() -> Result<()> {
         use rocket::http::{ContentType, Header};