From a5e504bfb71dbc515158d88e3202c33f2d11a9e7 Mon Sep 17 00:00:00 2001 From: sujan kota Date: Wed, 9 Sep 2026 12:40:23 -0400 Subject: [PATCH 1/2] feat(sdk): stream the TDF manifest, remove the ~21.7M segment ceiling (DSPX-4584) The manifest was assembled entirely in memory on both paths, so a payload past ~21.7M segments (41 TiB at 2 MiB segments) hard-failed with OutOfMemoryError: Requested array size exceeds VM limit. - serialize the manifest straight into the zip entry and parse it from a Reader, so neither side materializes it as a Java String - add Manifest.Segments, a compact fixed-stride AbstractList backed by packed hash bytes (~24 B/segment instead of ~176), with an ArrayList fallback for manifests this SDK did not write - materialize the aggregate hash once and share it with the root signature and every assertion instead of copying it per assertion Manifest JSON bytes are unchanged. The 0.manifest.json zip entry is now a streamed data-descriptor entry, the same form 0.payload already used. EOF --- .../io/opentdf/platform/sdk/Manifest.java | 213 +++++++++++++++++- .../java/io/opentdf/platform/sdk/SDK.java | 5 +- .../java/io/opentdf/platform/sdk/TDF.java | 93 +++++--- .../io/opentdf/platform/sdk/TDFReader.java | 27 ++- .../io/opentdf/platform/sdk/TDFWriter.java | 9 +- .../io/opentdf/platform/sdk/SegmentsTest.java | 213 ++++++++++++++++++ .../opentdf/platform/sdk/TDFWriterTest.java | 58 ++++- 7 files changed, 558 insertions(+), 60 deletions(-) create mode 100644 sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index 20fbb13e..b7ec6371 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -8,8 +8,14 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; import com.google.gson.annotations.JsonAdapter; import com.google.gson.annotations.SerializedName; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.JWSHeader; @@ -30,6 +36,7 @@ import org.erdtman.jcs.JsonCanonicalizer; import java.io.IOException; +import java.io.StringReader; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -38,6 +45,7 @@ import java.security.interfaces.RSAPublicKey; import java.security.cert.X509Certificate; import java.text.ParseException; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -118,6 +126,192 @@ public int hashCode() { } } + /** + * A compact, append-only {@link Segment} list. + *

+ * A plain {@code ArrayList} costs roughly 176 bytes per segment, which + * is several gigabytes for a multi-terabyte payload. Every segment of a TDF but + * the last has the same sizes, and every segment hash is the same fixed length, + * so all this stores is the hash characters packed into fixed-stride chunks + * plus the two size pairs — about 24 bytes per segment, with no array-growth + * copies and no {@code Integer.MAX_VALUE} ceiling on the backing store. + *

+ * {@link #append} refuses anything that does not fit those assumptions (a + * manifest from another writer may not honor them); callers fall back to an + * {@code ArrayList}. + *

+ * ponytail: the remaining ceiling is {@link #aggregate}, which materializes one + * byte array of {@code count * hashLength} — about 134M segments (256 TiB at + * 2 MiB segments), or ~100M if the TDF carries assertions, whose signature is + * base64 of that array. Going past that needs a streaming Mac and streaming + * base64, and a wire-format change for assertions. + */ + static final class Segments extends AbstractList { + private static final int SEGMENTS_PER_CHUNK = 4096; + + private final List chunks = new ArrayList<>(); + private int count; + /** Length of every hash, in characters; established by the first append. */ + private int stride; + /** Trailing '=' in every hash, so decoded hashes are also fixed length. */ + private int padding; + private long defaultSegmentSize; + private long defaultEncryptedSegmentSize; + private long lastSegmentSize; + private long lastEncryptedSegmentSize; + + /** + * @return false if this segment cannot be represented compactly, in which + * case the segment was not appended and the caller must fall back + */ + boolean append(String hash, long segmentSize, long encryptedSegmentSize) { + if (hash == null) { + return false; + } + if (count == 0) { + stride = hash.length(); + padding = trailingPadding(hash); + defaultSegmentSize = segmentSize; + defaultEncryptedSegmentSize = encryptedSegmentSize; + } else if (hash.length() != stride + || trailingPadding(hash) != padding + // the previously appended segment is no longer the last one, so it + // has to match the defaults from here on + || lastSegmentSize != defaultSegmentSize + || lastEncryptedSegmentSize != defaultEncryptedSegmentSize) { + return false; + } + + if (count % SEGMENTS_PER_CHUNK == 0) { + chunks.add(new byte[stride * SEGMENTS_PER_CHUNK]); + } + byte[] chunk = chunks.get(count / SEGMENTS_PER_CHUNK); + int offset = (count % SEGMENTS_PER_CHUNK) * stride; + for (int i = 0; i < stride; i++) { + char c = hash.charAt(i); + if (c > 0x7f) { // non-ASCII would not survive the byte-per-character packing + return false; + } + chunk[offset + i] = (byte) c; + } + + lastSegmentSize = segmentSize; + lastEncryptedSegmentSize = encryptedSegmentSize; + count++; + return true; + } + + private static int trailingPadding(String hash) { + int padding = 0; + while (padding < hash.length() && hash.charAt(hash.length() - 1 - padding) == '=') { + padding++; + } + return padding; + } + + /** + * Every segment hash concatenated, base64-decoded if {@code decodeBase64} + * and taken as raw ASCII otherwise. This is the aggregate hash that the root + * signature and the assertion signatures are computed over. + */ + byte[] aggregate(boolean decodeBase64) { + if (count == 0) { + return new byte[0]; + } + byte[] scratch = new byte[stride]; + if (!decodeBase64) { + byte[] aggregate = new byte[Math.multiplyExact(count, stride)]; + for (int i = 0; i < count; i++) { + copyHash(i, aggregate, i * stride); + } + return aggregate; + } + + copyHash(0, scratch, 0); + int decodedStride = Base64.getDecoder().decode(scratch).length; + byte[] aggregate = new byte[Math.multiplyExact(count, decodedStride)]; + byte[] decoded = new byte[decodedStride]; + for (int i = 0; i < count; i++) { + copyHash(i, scratch, 0); + Base64.getDecoder().decode(scratch, decoded); + System.arraycopy(decoded, 0, aggregate, i * decodedStride, decodedStride); + } + return aggregate; + } + + private void copyHash(int index, byte[] destination, int destinationOffset) { + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + System.arraycopy(chunk, (index % SEGMENTS_PER_CHUNK) * stride, destination, destinationOffset, stride); + } + + @Override + public Segment get(int index) { + Objects.checkIndex(index, count); + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + Segment segment = new Segment(); + segment.hash = new String(chunk, (index % SEGMENTS_PER_CHUNK) * stride, stride, StandardCharsets.US_ASCII); + boolean last = index == count - 1; + segment.segmentSize = last ? lastSegmentSize : defaultSegmentSize; + segment.encryptedSegmentSize = last ? lastEncryptedSegmentSize : defaultEncryptedSegmentSize; + return segment; + } + + @Override + public int size() { + return count; + } + } + + /** + * Reads {@code segments} into a {@link Segments} when the manifest allows it, + * and into a plain {@code ArrayList} when it does not. Writing delegates to the + * reflective {@link Segment} adapter, so the JSON is unchanged either way. + */ + static final class SegmentsAdapterFactory implements TypeAdapterFactory { + @Override + @SuppressWarnings("unchecked") + public TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter element = gson.getAdapter(Segment.class); + return (TypeAdapter) new TypeAdapter>() { + @Override + public void write(JsonWriter out, List value) throws IOException { + if (value == null) { + out.nullValue(); + return; + } + out.beginArray(); + for (Segment segment : value) { + element.write(out, segment); + } + out.endArray(); + } + + @Override + public List read(JsonReader in) throws IOException { + if (in.peek() == JsonToken.NULL) { + in.nextNull(); + return null; + } + in.beginArray(); + Segments compact = new Segments(); + List fallback = null; + while (in.hasNext()) { + Segment segment = element.read(in); + if (fallback != null) { + fallback.add(segment); + } else if (segment == null + || !compact.append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize)) { + fallback = new ArrayList<>(compact); + fallback.add(segment); + } + } + in.endArray(); + return fallback == null ? compact : fallback; + } + }; + } + } + static public class RootSignature { @SerializedName(value = "alg") public String algorithm; @@ -145,6 +339,7 @@ static public class IntegrityInformation { public String segmentHashAlg; public int segmentSizeDefault; public int encryptedSegmentSizeDefault; + @JsonAdapter(SegmentsAdapterFactory.class) public List segments; @Override @@ -559,6 +754,14 @@ public AssertionConfig.Statement deserialize(JsonElement json, Type typeOfT, Jso public Payload payload; public List assertions = new ArrayList<>(); protected static Manifest readManifest(String manifestJson) { + return readManifest(new StringReader(manifestJson)); + } + + /** + * Parses a manifest without materializing it as a {@link String}, which a + * manifest with tens of millions of segments cannot be. + */ + protected static Manifest readManifest(java.io.Reader manifestJson) { Manifest result = gson.fromJson(manifestJson, Manifest.class); if (result.assertions == null) { result.assertions = new ArrayList<>(); @@ -583,9 +786,13 @@ protected static Manifest readManifest(String manifestJson) { throw new IllegalArgumentException("Manifest with null policy"); } - for (Manifest.Segment segment : result.encryptionInformation.integrityInformation.segments) { - if (segment == null || segment.hash == null) { - throw new IllegalArgumentException("Invalid integrity segment"); + // Segments rejects null segments and hashes as it is built, so only the + // fallback representation needs checking here. + if (!(result.encryptionInformation.integrityInformation.segments instanceof Segments)) { + for (Manifest.Segment segment : result.encryptionInformation.integrityInformation.segments) { + if (segment == null || segment.hash == null) { + throw new IllegalArgumentException("Invalid integrity segment"); + } } } for (Manifest.KeyAccess keyAccess : result.encryptionInformation.keyAccessObj) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index 5e903498..272bf8be 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -185,8 +185,9 @@ public static boolean isTDF(SeekableByteChannel channel) { */ public static Manifest readManifest(SeekableByteChannel tdfBytes) throws SDKException, IOException { TDFReader reader = new TDFReader(tdfBytes); - String manifestJson = reader.manifest(); - return Manifest.readManifest(manifestJson); + try (var manifestJson = reader.manifest()) { + return Manifest.readManifest(manifestJson); + } } /** diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index b54901b9..775ae819 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -15,10 +15,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.security.*; @@ -81,6 +83,13 @@ private static byte[] tdfECKeySaltCompute() { private static final Gson gson = new GsonBuilder().create(); + /** + * The manifest is serialized directly into the zip entry, and + * {@link ZipWriter#stream} updates its CRC on every {@code write} call, so the + * character encoder needs a buffer in front of it. + */ + private static final int MANIFEST_WRITE_BUFFER_SIZE = 1 << 16; + /** * A self-imposed ceiling on the number of AES-GCM authenticated-encryption * invocations under a single payload key. One invocation is spent on the @@ -472,6 +481,23 @@ public PolicyObject readPolicyObject() { } } + /** + * Concatenates every segment hash, which is what the root signature and the + * assertion signatures are computed over. + */ + private static byte[] aggregateSegmentHashes(List segments, boolean isEncrypted) { + if (segments instanceof Manifest.Segments) { + return ((Manifest.Segments) segments).aggregate(isEncrypted); + } + var aggregateHash = new ByteArrayOutputStream(); + for (Manifest.Segment segment : segments) { + aggregateHash.writeBytes(isEncrypted + ? Base64.getDecoder().decode(segment.hash) + : segment.hash.getBytes()); + } + return aggregateHash.toByteArray(); + } + private static byte[] calculateSignature(byte[] data, byte[] secret, Config.IntegrityAlgorithm algorithm) { if (algorithm == Config.IntegrityAlgorithm.HS256) { return CryptoUtils.CalculateSHA256Hmac(secret, data); @@ -501,11 +527,11 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo long encryptedSegmentSize = (long) tdfConfig.defaultSegmentSize + kGcmIvSize + AesGcm.GCM_TAG_LENGTH; TDFWriter tdfWriter = new TDFWriter(outputStream); - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); byte[] readBuf = new byte[tdfConfig.defaultSegmentSize]; IvCounter payloadIv = IvCounter.forPayload(); - tdfObject.manifest.encryptionInformation.integrityInformation.segments = new ArrayList<>(); + var segments = new Manifest.Segments(); + tdfObject.manifest.encryptionInformation.integrityInformation.segments = segments; boolean finished; try (var payloadOutput = tdfWriter.payload()) { do { @@ -519,7 +545,6 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo byte[] cipherData; byte[] segmentSig; - Manifest.Segment segmentInfo = new Manifest.Segment(); // encrypt cipherData = tdfObject.aesGcm.encrypt(payloadIv.next(), AesGcm.GCM_TAG_LENGTH, @@ -530,19 +555,23 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo if (tdfConfig.hexEncodeRootAndSegmentHashes) { segmentSig = Hex.encodeHexString(segmentSig).getBytes(StandardCharsets.UTF_8); } - segmentInfo.hash = Base64.getEncoder().encodeToString(segmentSig); - aggregateHash.write(segmentSig); - segmentInfo.segmentSize = readThisLoop; - segmentInfo.encryptedSegmentSize = cipherData.length; - - tdfObject.manifest.encryptionInformation.integrityInformation.segments.add(segmentInfo); + // Every segment we write is full except the last one, and every hash has the + // same length, so the compact representation always accepts them. + if (!segments.append(Base64.getEncoder().encodeToString(segmentSig), readThisLoop, + cipherData.length)) { + throw new SDKException("unable to record segment " + segments.size() + " in the manifest"); + } } while (!finished); } + // Materialized once and reused by the root signature and every assertion below; + // it is proportional to the number of segments. + byte[] aggregateHash = segments.aggregate(true); + Manifest.RootSignature rootSignature = new Manifest.RootSignature(); - byte[] rootSig = calculateSignature(aggregateHash.toByteArray(), tdfObject.payloadKey, + byte[] rootSig = calculateSignature(aggregateHash, tdfObject.payloadKey, tdfConfig.integrityAlgorithm); byte[] encodedRootSig = tdfConfig.hexEncodeRootAndSegmentHashes ? Hex.encodeHexString(rootSig).getBytes(StandardCharsets.UTF_8) @@ -595,9 +624,9 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo throw new SDKException("error decoding assertion hash", e); } } - byte[] completeHash = new byte[aggregateHash.size() + assertionHash.length]; - System.arraycopy(aggregateHash.toByteArray(), 0, completeHash, 0, aggregateHash.size()); - System.arraycopy(assertionHash, 0, completeHash, aggregateHash.size(), assertionHash.length); + byte[] completeHash = new byte[aggregateHash.length + assertionHash.length]; + System.arraycopy(aggregateHash, 0, completeHash, 0, aggregateHash.length); + System.arraycopy(assertionHash, 0, completeHash, aggregateHash.length, assertionHash.length); var encodedHash = Base64.getEncoder().encodeToString(completeHash); @@ -618,9 +647,13 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo } tdfObject.manifest.assertions = signedAssertions; - String manifestAsStr = gson.toJson(tdfObject.manifest); - tdfWriter.appendManifest(manifestAsStr); + // Serialize straight into the zip entry: a manifest with tens of millions of segments + // exceeds the maximum size of a Java String. + try (var manifestOutput = new BufferedWriter( + new OutputStreamWriter(tdfWriter.manifest(), StandardCharsets.UTF_8), MANIFEST_WRITE_BUFFER_SIZE)) { + gson.toJson(tdfObject.manifest, manifestOutput); + } tdfObject.size = tdfWriter.finish(); return tdfObject; @@ -657,9 +690,11 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig, Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throws SDKException, IOException { TDFReader tdfReader = new TDFReader(tdf); - String manifestJson = tdfReader.manifest(); // use Manifest.readManifest in order to validate the Manifest input - Manifest manifest = Manifest.readManifest(manifestJson); + Manifest manifest; + try (var manifestJson = tdfReader.manifest()) { + manifest = Manifest.readManifest(manifestJson); + } byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; @@ -744,15 +779,8 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) String rootAlgorithm = manifest.encryptionInformation.integrityInformation.rootSignature.algorithm; String rootSignature = manifest.encryptionInformation.integrityInformation.rootSignature.signature; - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { - if (manifest.payload.isEncrypted) { - byte[] decodedHash = Base64.getDecoder().decode(segment.hash); - aggregateHash.write(decodedHash); - } else { - aggregateHash.write(segment.hash.getBytes()); - } - } + byte[] aggregateHash = aggregateSegmentHashes( + manifest.encryptionInformation.integrityInformation.segments, manifest.payload.isEncrypted); String rootSigValue; boolean isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); @@ -762,7 +790,7 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) sigAlg = Config.IntegrityAlgorithm.GMAC; } - var sig = calculateSignature(aggregateHash.toByteArray(), payloadKey, sigAlg); + var sig = calculateSignature(aggregateHash, payloadKey, sigAlg); if (isLegacyTdf) { sig = Hex.encodeHexString(sig).getBytes(); } @@ -775,7 +803,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new IllegalStateException("error getting instance of SHA-256 digest", e); } - rootSigValue = Base64.getEncoder().encodeToString(digest.digest(aggregateHash.toString().getBytes())); + // the round trip through the platform default charset is a no-op for the + // hex hashes this branch sees, and is kept so the digest is unchanged + rootSigValue = Base64.getEncoder().encodeToString(digest.digest(new String(aggregateHash).getBytes())); } if (rootSignature.compareTo(rootSigValue) != 0) { @@ -790,7 +820,6 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) "segment size mismatch. encrypted segment size differs from plaintext segment size. the TDF is invalid"); } - var aggregateHashByteArrayBytes = aggregateHash.toByteArray(); // Validate assertions for (var assertion : manifest.assertions) { // Skip assertion verification if disabled @@ -830,9 +859,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new SDKException("error decoding assertion hash", e); } } - var signature = new byte[aggregateHashByteArrayBytes.length + hashOfAssertion.length]; - System.arraycopy(aggregateHashByteArrayBytes, 0, signature, 0, aggregateHashByteArrayBytes.length); - System.arraycopy(hashOfAssertion, 0, signature, aggregateHashByteArrayBytes.length, hashOfAssertion.length); + var signature = new byte[aggregateHash.length + hashOfAssertion.length]; + System.arraycopy(aggregateHash, 0, signature, 0, aggregateHash.length); + System.arraycopy(hashOfAssertion, 0, signature, aggregateHash.length, hashOfAssertion.length); var encodeSignature = Base64.getEncoder().encodeToString(signature); if (!Objects.equals(encodeSignature, hashValues.getSignature())) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index 6e9f32d2..ca068e45 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -1,8 +1,10 @@ package io.opentdf.platform.sdk; -import java.io.ByteArrayOutputStream; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -18,6 +20,8 @@ */ public class TDFReader { + private static final int MANIFEST_BUFFER_SIZE = 1 << 16; + private final ZipReader.Entry manifestEntry; private final InputStream payload; @@ -37,15 +41,18 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { payload = entries.get(TDF_PAYLOAD_FILE_NAME).getData(); } - String manifest() { - var out = new ByteArrayOutputStream(); + /** + * The manifest entry as a character stream; the caller must close it. Returned + * as a stream rather than a String because a manifest with tens of millions of + * segments exceeds the maximum size of a Java String. + */ + Reader manifest() { try { - manifestEntry.getData().transferTo(out); + return new BufferedReader( + new InputStreamReader(manifestEntry.getData(), StandardCharsets.UTF_8), MANIFEST_BUFFER_SIZE); } catch (IOException e) { throw new SDKException("error retrieving manifest from zip file", e); } - - return out.toString(StandardCharsets.UTF_8); } int readPayloadBytes(byte[] buf) { @@ -62,8 +69,10 @@ int readPayloadBytes(byte[] buf) { } PolicyObject readPolicyObject() { - String manifestJson = manifest(); - Manifest manifest = Manifest.readManifest(manifestJson); - return Manifest.decodePolicyObject(manifest); + try (Reader manifestJson = manifest()) { + return Manifest.decodePolicyObject(Manifest.readManifest(manifestJson)); + } catch (IOException e) { + throw new SDKException("error reading manifest from zip file", e); + } } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java index 7137c232..06357c2b 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; /** * The TDFWriter class provides functionalities for creating a TDF (Trusted Data Format) archive. @@ -24,8 +23,12 @@ public TDFWriter(OutputStream destination) { this.archiveWriter = new ZipWriter(destination, maxNonZip64Value); } - public void appendManifest(String manifest) throws IOException { - this.archiveWriter.data(TDF_MANIFEST_FILE_NAME, manifest.getBytes(StandardCharsets.UTF_8)); + /** + * Opens the manifest entry for writing. The returned stream must be closed before + * {@link #finish()} is called, otherwise the entry never makes it into the central directory. + */ + public OutputStream manifest() throws IOException { + return this.archiveWriter.stream(TDF_MANIFEST_FILE_NAME); } public OutputStream payload() throws IOException { diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java new file mode 100644 index 00000000..ef80ce72 --- /dev/null +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -0,0 +1,213 @@ +package io.opentdf.platform.sdk; + +import com.google.gson.Gson; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Covers the compact {@link Manifest.Segments} representation: it has to + * serialize exactly like a plain list, fall back when a manifest does not fit + * its assumptions, and let a manifest larger than a Java String round trip. + */ +public class SegmentsTest { + + private static final Gson GSON = new Gson(); + private static final int SEGMENT_SIZE = 16384; + private static final int ENCRYPTED_SEGMENT_SIZE = SEGMENT_SIZE + 28; + + private static String hash(int i) { + var bytes = new byte[16]; + bytes[0] = (byte) i; + bytes[1] = (byte) (i >> 8); + bytes[2] = (byte) (i >> 16); + return Base64.getEncoder().encodeToString(bytes); + } + + private static Manifest.Segment segment(String hash, long segmentSize, long encryptedSegmentSize) { + var segment = new Manifest.Segment(); + segment.hash = hash; + segment.segmentSize = segmentSize; + segment.encryptedSegmentSize = encryptedSegmentSize; + return segment; + } + + private static String manifestJson(List segments) { + return "{\"schemaVersion\":\"4.3.0\",\"encryptionInformation\":{\"type\":\"split\"," + // no '=' padding: Gson HTML-escapes it, and this literal is compared verbatim + + "\"policy\":\"eyJib2R5Ijp7fX0K\",\"keyAccess\":[{\"type\":\"wrapped\",\"url\":\"http://kas\"," + + "\"protocol\":\"kas\",\"wrappedKey\":\"AAAA\"}]," + + "\"method\":{\"algorithm\":\"AES-256-GCM\",\"iv\":\"AAAAAAAAAAAA\",\"isStreamable\":true}," + + "\"integrityInformation\":{\"rootSignature\":{\"alg\":\"GMAC\",\"sig\":\"AAAA\"}," + + "\"segmentHashAlg\":\"GMAC\",\"segmentSizeDefault\":" + SEGMENT_SIZE + + ",\"encryptedSegmentSizeDefault\":" + ENCRYPTED_SEGMENT_SIZE + ",\"segments\":" + + segments.stream().map(GSON::toJson).collect(Collectors.joining(",", "[", "]")) + + "}},\"payload\":{\"type\":\"reference\",\"url\":\"0.payload\",\"protocol\":\"zip\"," + + "\"mimeType\":\"application/octet-stream\",\"isEncrypted\":true},\"assertions\":[]}"; + } + + /** A run of uniform segments with a short final one, i.e. what this SDK writes. */ + private static List uniformSegments(int count) { + return IntStream.range(0, count) + .mapToObj(i -> segment(hash(i), + i == count - 1 ? 7 : SEGMENT_SIZE, + i == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)) + .collect(Collectors.toList()); + } + + @Test + void uniformSegmentsAreStoredCompactlyAndSerializeUnchanged() { + var segments = uniformSegments(5); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void irregularSegmentSizesFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + // a non-final segment that disagrees with the defaults cannot be stored compactly + segments.set(1, segment(hash(1), SEGMENT_SIZE / 2, ENCRYPTED_SEGMENT_SIZE / 2)); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void segmentsWithDifferingHashLengthsFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + segments.set(1, segment("deadbeef", SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + var json = manifestJson(segments); + + var parsed = Manifest.readManifest(json).encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + } + + @Test + void aggregateConcatenatesEveryHash() { + var segments = new Manifest.Segments(); + for (int i = 0; i < 3; i++) { + assertThat(segments.append(hash(i), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)).isTrue(); + } + + var decoded = new byte[48]; + for (int i = 0; i < 3; i++) { + System.arraycopy(Base64.getDecoder().decode(hash(i)), 0, decoded, i * 16, 16); + } + assertThat(segments.aggregate(true)).isEqualTo(decoded); + assertThat(segments.aggregate(false)) + .isEqualTo((hash(0) + hash(1) + hash(2)).getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + @Test + void emptySegmentsAggregateToNothing() { + assertThat(new Manifest.Segments().aggregate(true)).isEmpty(); + assertThat(new Manifest.Segments().aggregate(false)).isEmpty(); + } + + /** + * A manifest whose JSON is larger than {@code Integer.MAX_VALUE} cannot be held + * as a String, so both directions have to stream. Opt in with + * {@code -Dtdf.hugeSegments=true -Xmx2g}; it needs roughly a minute and 1 GiB. + */ + @Test + @EnabledIfSystemProperty(named = "tdf.hugeSegments", matches = "true") + void aManifestTooLargeToBeAStringRoundTrips() throws IOException { + // ~25M segments at ~99 JSON bytes each is ~2.5 GB, past the String ceiling + final int count = 25_000_000; + + Manifest manifest; + try (Reader json = new GeneratedManifestReader(count)) { + manifest = Manifest.readManifest(json); + } + var segments = manifest.encryptionInformation.integrityInformation.segments; + assertThat(segments).isInstanceOf(Manifest.Segments.class); + assertThat(segments.size()).isEqualTo(count); + assertThat(segments.get(count - 1).hash).isEqualTo(hash(count - 1)); + + var counting = new CountingWriter(); + new Gson().toJson(manifest, counting); + assertThat(counting.written).isGreaterThan(Integer.MAX_VALUE); + } + + /** Renders a manifest with {@code count} segments without ever storing it. */ + private static final class GeneratedManifestReader extends Reader { + private final int count; + private final String suffix; + private int next; + private String pending; + private int pendingOffset; + + GeneratedManifestReader(int count) { + this.count = count; + var template = manifestJson(uniformSegments(1)); + this.pending = template.substring(0, template.indexOf("\"segments\":[") + "\"segments\":[".length()); + this.suffix = template.substring(template.indexOf("]}},\"payload\"")); + } + + @Override + public int read(char[] buffer, int offset, int length) { + if (pendingOffset == pending.length()) { + if (next > count) { + return -1; + } + pending = next == count ? suffix + : (next == 0 ? "" : ",") + GSON.toJson(segment(hash(next), + next == count - 1 ? 7 : SEGMENT_SIZE, + next == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)); + pendingOffset = 0; + next++; + if (pending.isEmpty()) { + return read(buffer, offset, length); + } + } + int n = Math.min(length, pending.length() - pendingOffset); + pending.getChars(pendingOffset, pendingOffset + n, buffer, offset); + pendingOffset += n; + return n; + } + + @Override + public void close() { + } + } + + private static final class CountingWriter extends Writer { + private long written; + + @Override + public void write(char[] buffer, int offset, int length) { + written += length; + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + } +} diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java index 2f5d22fc..84433a88 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java @@ -5,13 +5,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.StringWriter; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.StandardOpenOption; +import java.util.stream.Collectors; +import java.util.zip.ZipFile; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class TDFWriterTest { @Test @@ -64,14 +69,39 @@ void simpleTDFCreate() throws IOException { " }\n" + "}"; String payload = "Hello, world!"; - FileOutputStream fileOutStream = new FileOutputStream("sample.tdf"); - TDFWriter writer = new TDFWriter(fileOutStream); - try (var p = writer.payload()) { - new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + var tdfFile = File.createTempFile("sample", ".tdf"); + tdfFile.deleteOnExit(); + try (var fileOutStream = new FileOutputStream(tdfFile)) { + TDFWriter writer = new TDFWriter(fileOutStream); + try (var p = writer.payload()) { + new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + } + try (var m = writer.manifest()) { + m.write(kManifestJsonFromTDF.getBytes(StandardCharsets.UTF_8)); + } + writer.finish(); + } + + // our own reader + try (var channel = FileChannel.open(tdfFile.toPath(), StandardOpenOption.READ)) { + var entries = new ZipReader(channel).getEntries().stream() + .collect(Collectors.toMap(ZipReader.Entry::getName, e -> e)); + assertEquals(kManifestJsonFromTDF, + new String(entries.get(TDFWriter.TDF_MANIFEST_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + assertEquals(payload, + new String(entries.get(TDFWriter.TDF_PAYLOAD_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + } + + // an independent central-directory based reader, as a stand-in for the other SDKs + try (var zipFile = new ZipFile(tdfFile)) { + var manifestEntry = zipFile.getEntry(TDFWriter.TDF_MANIFEST_FILE_NAME); + assertNotNull(manifestEntry); + try (var in = zipFile.getInputStream(manifestEntry)) { + assertEquals(kManifestJsonFromTDF, new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } } - writer.appendManifest(kManifestJsonFromTDF); - writer.finish(); - fileOutStream.close(); } /** @@ -89,12 +119,18 @@ void readsBackAManifestWrittenPastTheZip64Boundary() throws IOException { try (var p = writer.payload()) { new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); } - writer.appendManifest(manifest); + try (var m = writer.manifest()) { + m.write(manifest.getBytes(StandardCharsets.UTF_8)); + } writer.finish(); try (var chan = new SeekableInMemoryByteChannel(out.toByteArray())) { var reader = new TDFReader(chan); - assertEquals(manifest, reader.manifest()); + var readBack = new StringWriter(); + try (var m = reader.manifest()) { + m.transferTo(readBack); + } + assertEquals(manifest, readBack.toString()); var payloadBytes = new byte[payload.length()]; assertEquals(payload.length(), reader.readPayloadBytes(payloadBytes)); From 4d084123caf648f04015679f86f4a1b94d4dcd58 Mon Sep 17 00:00:00 2001 From: sujan kota Date: Fri, 11 Sep 2026 13:31:50 -0400 Subject: [PATCH 2/2] fix(sdk): bound segment hash stride, keep appendManifest as deprecated --- .../java/io/opentdf/platform/sdk/Manifest.java | 11 +++++++++++ .../java/io/opentdf/platform/sdk/TDFReader.java | 4 ++-- .../java/io/opentdf/platform/sdk/TDFWriter.java | 13 +++++++++++++ .../java/io/opentdf/platform/sdk/SegmentsTest.java | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index b7ec6371..1a3de467 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -148,6 +148,14 @@ public int hashCode() { */ static final class Segments extends AbstractList { private static final int SEGMENTS_PER_CHUNK = 4096; + /** + * A chunk is {@code stride * SEGMENTS_PER_CHUNK} bytes, so an absurd hash in an + * untrusted manifest would size the very first allocation. The longest hash any + * writer produces is 128 characters (hex-encoded HS256); anything past this is + * left to the {@code ArrayList} fallback, which only ever costs what the input + * itself costs. + */ + private static final int MAX_HASH_LENGTH = 1024; private final List chunks = new ArrayList<>(); private int count; @@ -169,6 +177,9 @@ boolean append(String hash, long segmentSize, long encryptedSegmentSize) { return false; } if (count == 0) { + if (hash.isEmpty() || hash.length() > MAX_HASH_LENGTH) { + return false; + } stride = hash.length(); padding = trailingPadding(hash); defaultSegmentSize = segmentSize; diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index ca068e45..590a17c3 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -69,8 +69,8 @@ int readPayloadBytes(byte[] buf) { } PolicyObject readPolicyObject() { - try (Reader manifestJson = manifest()) { - return Manifest.decodePolicyObject(Manifest.readManifest(manifestJson)); + try (Reader manifestRaw = manifest()) { + return Manifest.decodePolicyObject(Manifest.readManifest(manifestRaw)); } catch (IOException e) { throw new SDKException("error reading manifest from zip file", e); } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java index 06357c2b..b64a39f1 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; /** * The TDFWriter class provides functionalities for creating a TDF (Trusted Data Format) archive. @@ -23,6 +24,18 @@ public TDFWriter(OutputStream destination) { this.archiveWriter = new ZipWriter(destination, maxNonZip64Value); } + /** + * @deprecated use {@link #manifest()}. A manifest with tens of millions of segments + * exceeds the maximum size of a Java {@link String}, so this cannot + * express every manifest the SDK writes. It produces the same zip entry. + */ + @Deprecated + public void appendManifest(String manifest) throws IOException { + try (OutputStream output = manifest()) { + output.write(manifest.getBytes(StandardCharsets.UTF_8)); + } + } + /** * Opens the manifest entry for writing. The returned stream must be closed before * {@link #finish()} is called, otherwise the entry never makes it into the central directory. diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java index ef80ce72..3c5155ee 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -105,6 +105,20 @@ void segmentsWithDifferingHashLengthsFallBackToAPlainList() { assertThat(parsed).containsExactlyElementsOf(segments); } + @Test + void anAbsurdlyLongHashFallsBackInsteadOfSizingAChunkFromIt() { + // a chunk is stride * 4096 bytes, so without a bound this one hash would ask for + // gigabytes before the fallback ever ran + var segments = new ArrayList<>(uniformSegments(2)); + segments.set(0, segment("A".repeat(100_000), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + + var parsed = Manifest.readManifest(manifestJson(segments)) + .encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + } + @Test void aggregateConcatenatesEveryHash() { var segments = new Manifest.Segments();