feat(tdf): spec-compliant container — manifest.json entry, payload from payload.url

packages/otdf-python/README.md

@@ -97,6 +97,12 @@ with open("decrypted.txt", "wb") as f:
 
 ```
 
+## TDF container format
+
+TDF archives are written with the spec-mandated `manifest.json` zip entry (per opentdf/spec) instead of the legacy `0.manifest.json`. Readers accept both names, so archives written by older versions of this SDK remain readable forever. The payload zip entry is resolved from `manifest.payload.url` rather than assumed. `schemaVersion` is unchanged (still `4.3.0`); `tdf_spec_version` is now read at either its legacy or spec-conformant placement, but never written.
+
+**Interoperability note:** archives written by this version cannot currently be opened by released `otdfctl`, or by the upstream `opentdf/platform` Go, Java, or JS SDKs, until they add a `manifest.json` read fallback (an upstream reader-fallback PR is planned). If you interoperate with those tools today, hold off upgrading until that fallback ships.
+
 ## Project Structure
 
 ```

packages/otdf-python/src/otdf_python/cli.py

@@ -205,12 +205,15 @@ def build_sdk(args) -> SDK:
 
 # Features the Python SDK currently exercises honestly in community xtest Stage-1.
 # Keep conservative: only advertise what encrypt/decrypt paths actually honor.
+# "spec-container": writes manifest.json at the zip root and resolves the
+# payload entry from manifest.payload.url (opentdf/spec container rules).
 _SUPPORTED_FEATURES: frozenset[str] = frozenset(
     {
         "autoconfigure",
         "connectrpc",
         "hexless",
         "kasallowlist",
+        "spec-container",
     }
 )
 
@@ -238,6 +241,7 @@ def build_sdk(args) -> SDK:
         "mechanism-mlkem",
         "ns_grants",
         "obligations",
+        "spec-container",
     }
 )
 

packages/otdf-python/src/otdf_python/manifest.py

@@ -86,6 +86,8 @@ class ManifestPayload:
     protocol: str
     mimeType: str
     isEncrypted: bool
+    # Read-only: some spec revisions place the version here. Never written.
+    tdf_spec_version: str | None = None
 
 
 @dataclass
@@ -116,6 +118,18 @@ class Manifest:
     encryptionInformation: ManifestEncryptionInformation | None = None
     payload: ManifestPayload | None = None
     assertions: list[ManifestAssertion] = field(default_factory=list)
+    # Read-only: spec prose places tdf_spec_version at the top level. Never written.
+    tdf_spec_version: str | None = None
+
+    def spec_version(self) -> str | None:
+        """Resolve the spec version: schemaVersion, then tdf_spec_version, then payload.tdf_spec_version."""
+        if self.schemaVersion:
+            return self.schemaVersion
+        if self.tdf_spec_version:
+            return self.tdf_spec_version
+        if self.payload and self.payload.tdf_spec_version:
+            return self.payload.tdf_spec_version
+        return None
 
     def _remove_none_values_and_empty_lists(self, obj):
         """Recursively remove None values and empty lists from dictionaries and lists."""
@@ -147,7 +161,9 @@ def to_json(self) -> str:
             manifest_dict["encryptionInformation"] = asdict(self.encryptionInformation)
 
         if self.payload is not None:
-            manifest_dict["payload"] = asdict(self.payload)
+            payload_dict = asdict(self.payload)
+            payload_dict.pop("tdf_spec_version", None)
+            manifest_dict["payload"] = payload_dict
 
         if self.schemaVersion is not None:
             manifest_dict["schemaVersion"] = self.schemaVersion
@@ -229,6 +245,7 @@ def _assertion(a):
 
         return Manifest(
             schemaVersion=d.get("schemaVersion", d.get("tdf_version")),
+            tdf_spec_version=d.get("tdf_spec_version"),
             encryptionInformation=_enc_info(
                 d.get("encryptionInformation", d.get("encryption_information"))
             )

packages/otdf-python/src/otdf_python/sdk.py

@@ -365,16 +365,24 @@ def is_tdf(data: bytes | BinaryIO) -> bool:
             bool: True if the data is a TDF, False otherwise
 
         """
+        import json
         import zipfile
         from io import BytesIO
 
+        from otdf_python.tdf_reader import resolve_manifest_name, resolve_payload_name
+
         try:
             file_like = BytesIO(data) if isinstance(data, bytes | bytearray) else data
             with zipfile.ZipFile(file_like) as zf:
-                names = set(zf.namelist())
-                return {"0.manifest.json", "0.payload"}.issubset(names) and len(
-                    names
-                ) == 2
+                names = zf.namelist()
+                manifest_name = resolve_manifest_name(names)
+                manifest = json.loads(zf.read(manifest_name))
+                payload = (
+                    manifest.get("payload") if isinstance(manifest, dict) else None
+                )
+                url = payload.get("url") if isinstance(payload, dict) else None
+                resolve_payload_name(url, names)
+                return True
         except Exception:
             return False
 

packages/otdf-python/src/otdf_python/tdf.py

@@ -28,6 +28,7 @@
     ManifestSegment,
 )
 from otdf_python.policy_stub import NULL_POLICY_UUID
+from otdf_python.tdf_reader import resolve_manifest_name, resolve_payload_name
 from otdf_python.tdf_writer import TDFWriter
 
 
@@ -399,7 +400,7 @@ def create_tdf(
         )
         payload_info = ManifestPayload(
             type="reference",  # Changed from "file" to "reference" to match Java SDK
-            url="0.payload",
+            url=TDFWriter.TDF_PAYLOAD_FILE_NAME,
             protocol="zip",
             mimeType=config.mime_type,  # Use MIME type from config
             isEncrypted=True,  # Changed from is_encrypted to isEncrypted
@@ -433,7 +434,8 @@ def load_tdf(
         tdf_bytes_io = io.BytesIO(tdf_data) if isinstance(tdf_data, bytes) else tdf_data
 
         with zipfile.ZipFile(tdf_bytes_io, "r") as z:
-            manifest_json = z.read("0.manifest.json").decode()
+            names = z.namelist()
+            manifest_json = z.read(resolve_manifest_name(names)).decode()
             manifest = Manifest.from_json(manifest_json)
 
             if not manifest.encryptionInformation:
@@ -464,7 +466,8 @@ def load_tdf(
             segments = (
                 manifest.encryptionInformation.integrityInformation.segments
             )  # Changed field name
-            encrypted_payload = z.read("0.payload")
+            payload_url = manifest.payload.url if manifest.payload else None
+            encrypted_payload = z.read(resolve_payload_name(payload_url, names))
             payload = self._decrypt_segments(aesgcm, segments, encrypted_payload)
             return TDFReader(payload=payload, manifest=manifest)
 
@@ -487,7 +490,8 @@ def read_payload(
         from .asym_crypto import AsymDecryption
 
         with zipfile.ZipFile(io.BytesIO(tdf_bytes), "r") as z:
-            manifest_json = z.read("0.manifest.json").decode()
+            names = z.namelist()
+            manifest_json = z.read(resolve_manifest_name(names)).decode()
             manifest = Manifest.from_json(manifest_json)
 
             if not manifest.encryptionInformation:
@@ -510,7 +514,8 @@ def read_payload(
             segments = (
                 manifest.encryptionInformation.integrityInformation.segments
             )  # Changed field names
-            encrypted_payload = z.read("0.payload")
+            payload_url = manifest.payload.url if manifest.payload else None
+            encrypted_payload = z.read(resolve_payload_name(payload_url, names))
             offset = 0
             for seg in segments:
                 enc_len = seg.encryptedSegmentSize  # Changed field name

packages/otdf-python/src/otdf_python/tdf_reader.py

@@ -1,15 +1,71 @@
 """TDFReader is responsible for reading and processing Trusted Data Format (TDF) files."""
 
+import json
+from collections.abc import Iterable
+
 from .manifest import Manifest
 from .policy_object import PolicyObject
 from .sdk_exceptions import SDKException
 from .zip_reader import ZipReader
 
-# Constants from TDFWriter
-TDF_MANIFEST_FILE_NAME = "0.manifest.json"
+# Spec (opentdf/spec schema/OpenTDF/README.md): the manifest entry MUST be
+# `manifest.json` at the archive root. `0.manifest.json` is what every SDK
+# wrote before this change and is accepted forever on read.
+TDF_MANIFEST_FILE_NAME = "manifest.json"
+LEGACY_TDF_MANIFEST_FILE_NAME = "0.manifest.json"
+# Payload entry name. Written by TDFWriter and into manifest.payload.url.
+# On read this is only a fallback for manifests with an empty payload.url.
 TDF_PAYLOAD_FILE_NAME = "0.payload"
 
 
+def resolve_manifest_name(names: Iterable[str]) -> str:
+    """Return the zip entry holding the manifest, spec name first."""
+    name_set = set(names)
+    for candidate in (TDF_MANIFEST_FILE_NAME, LEGACY_TDF_MANIFEST_FILE_NAME):
+        if candidate in name_set:
+            return candidate
+    raise ValueError("tdf doesn't contain a manifest")
+
+
+def _is_safe_entry_name(name: str) -> bool:
+    if not name or name.startswith("/") or "\\" in name:
+        return False
+    return ".." not in name.split("/")
+
+
+def resolve_payload_name(payload_url: str | None, names: Iterable[str]) -> str:
+    """Return the zip entry holding the payload.
+
+    Uses manifest.payload.url when present; falls back to `0.payload` only
+    when the url is empty or missing.
+    """
+    name_set = set(names)
+    if payload_url:
+        if not _is_safe_entry_name(payload_url):
+            raise ValueError(f"unsafe payload url in manifest: {payload_url!r}")
+        if payload_url in name_set:
+            return payload_url
+        raise ValueError(f"tdf doesn't contain payload entry {payload_url!r}")
+    if TDF_PAYLOAD_FILE_NAME in name_set:
+        return TDF_PAYLOAD_FILE_NAME
+    raise ValueError("tdf doesn't contain a payload")
+
+
+def payload_url_from_manifest_json(manifest_text: str) -> str | None:
+    """Extract payload.url without requiring a fully valid manifest."""
+    try:
+        data = json.loads(manifest_text)
+    except (TypeError, ValueError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    payload = data.get("payload")
+    if not isinstance(payload, dict):
+        return None
+    url = payload.get("url")
+    return url if isinstance(url, str) else None
+
+
 class TDFReader:
     """TDFReader is responsible for reading and processing Trusted Data Format (TDF) files.
 
@@ -31,15 +87,16 @@ def __init__(self, tdf):
         try:
             self._zip_reader = ZipReader(tdf)
             namelist = self._zip_reader.namelist()
-
-            if TDF_MANIFEST_FILE_NAME not in namelist:
-                raise ValueError("tdf doesn't contain a manifest")
-            if TDF_PAYLOAD_FILE_NAME not in namelist:
-                raise ValueError("tdf doesn't contain a payload")
-
-            # Store the names for later use
-            self._manifest_name = TDF_MANIFEST_FILE_NAME
-            self._payload_name = TDF_PAYLOAD_FILE_NAME
+            self._manifest_name = resolve_manifest_name(namelist)
+            manifest_text = self._zip_reader.read(self._manifest_name).decode("utf-8")
+            payload_url = payload_url_from_manifest_json(manifest_text)
+            self._payload_name = resolve_payload_name(payload_url, namelist)
+        except UnicodeDecodeError as e:
+            # UnicodeDecodeError is a ValueError subclass, but it means the
+            # manifest entry is corrupt, not that the tdf is missing an
+            # entry (the resolvers' own ValueErrors). Wrap it like any
+            # other unexpected failure instead of letting it pass through.
+            raise SDKException("Error initializing TDFReader") from e
         except Exception as e:
             if isinstance(e, ValueError):
                 raise

packages/otdf-python/src/otdf_python/tdf_writer.py

@@ -8,8 +8,10 @@
 class TDFWriter:
     """TDF file writer for creating encrypted TDF packages."""
 
+    # Spec: manifest entry MUST be `manifest.json` at the archive root.
+    TDF_MANIFEST_FILE_NAME = "manifest.json"
+    # Payload entry name; TDF.create_tdf writes this same value into manifest.payload.url.
     TDF_PAYLOAD_FILE_NAME = "0.payload"
-    TDF_MANIFEST_FILE_NAME = "0.manifest.json"
 
     def __init__(self, out_stream: io.BytesIO | None = None):
         """Initialize TDF writer."""