diff --git a/packages/otdf-python/README.md b/packages/otdf-python/README.md index 88dd884a..ce27c462 100644 --- a/packages/otdf-python/README.md +++ b/packages/otdf-python/README.md @@ -97,6 +97,12 @@ with open("decrypted.txt", "wb") as f: ``` +## TDF container format + +TDF archives are written with the spec-mandated `manifest.json` zip entry (per opentdf/spec) instead of the legacy `0.manifest.json`. Readers accept both names, so archives written by older versions of this SDK remain readable forever. The payload zip entry is resolved from `manifest.payload.url` rather than assumed. `schemaVersion` is unchanged (still `4.3.0`); `tdf_spec_version` is now read at either its legacy or spec-conformant placement, but never written. + +**Interoperability note:** archives written by this version cannot currently be opened by released `otdfctl`, or by the upstream `opentdf/platform` Go, Java, or JS SDKs, until they add a `manifest.json` read fallback (an upstream reader-fallback PR is planned). If you interoperate with those tools today, hold off upgrading until that fallback ships. + ## Project Structure ``` diff --git a/packages/otdf-python/src/otdf_python/cli.py b/packages/otdf-python/src/otdf_python/cli.py index 6ddd13fd..80de6cda 100644 --- a/packages/otdf-python/src/otdf_python/cli.py +++ b/packages/otdf-python/src/otdf_python/cli.py @@ -205,12 +205,15 @@ def build_sdk(args) -> SDK: # Features the Python SDK currently exercises honestly in community xtest Stage-1. # Keep conservative: only advertise what encrypt/decrypt paths actually honor. +# "spec-container": writes manifest.json at the zip root and resolves the +# payload entry from manifest.payload.url (opentdf/spec container rules). _SUPPORTED_FEATURES: frozenset[str] = frozenset( { "autoconfigure", "connectrpc", "hexless", "kasallowlist", + "spec-container", } ) @@ -238,6 +241,7 @@ def build_sdk(args) -> SDK: "mechanism-mlkem", "ns_grants", "obligations", + "spec-container", } ) diff --git a/packages/otdf-python/src/otdf_python/manifest.py b/packages/otdf-python/src/otdf_python/manifest.py index 0cf7d4f6..faf7d755 100644 --- a/packages/otdf-python/src/otdf_python/manifest.py +++ b/packages/otdf-python/src/otdf_python/manifest.py @@ -86,6 +86,8 @@ class ManifestPayload: protocol: str mimeType: str isEncrypted: bool + # Read-only: some spec revisions place the version here. Never written. + tdf_spec_version: str | None = None @dataclass @@ -116,6 +118,18 @@ class Manifest: encryptionInformation: ManifestEncryptionInformation | None = None payload: ManifestPayload | None = None assertions: list[ManifestAssertion] = field(default_factory=list) + # Read-only: spec prose places tdf_spec_version at the top level. Never written. + tdf_spec_version: str | None = None + + def spec_version(self) -> str | None: + """Resolve the spec version: schemaVersion, then tdf_spec_version, then payload.tdf_spec_version.""" + if self.schemaVersion: + return self.schemaVersion + if self.tdf_spec_version: + return self.tdf_spec_version + if self.payload and self.payload.tdf_spec_version: + return self.payload.tdf_spec_version + return None def _remove_none_values_and_empty_lists(self, obj): """Recursively remove None values and empty lists from dictionaries and lists.""" @@ -147,7 +161,9 @@ def to_json(self) -> str: manifest_dict["encryptionInformation"] = asdict(self.encryptionInformation) if self.payload is not None: - manifest_dict["payload"] = asdict(self.payload) + payload_dict = asdict(self.payload) + payload_dict.pop("tdf_spec_version", None) + manifest_dict["payload"] = payload_dict if self.schemaVersion is not None: manifest_dict["schemaVersion"] = self.schemaVersion @@ -229,6 +245,7 @@ def _assertion(a): return Manifest( schemaVersion=d.get("schemaVersion", d.get("tdf_version")), + tdf_spec_version=d.get("tdf_spec_version"), encryptionInformation=_enc_info( d.get("encryptionInformation", d.get("encryption_information")) ) diff --git a/packages/otdf-python/src/otdf_python/sdk.py b/packages/otdf-python/src/otdf_python/sdk.py index 9945f3ab..63e7ad44 100644 --- a/packages/otdf-python/src/otdf_python/sdk.py +++ b/packages/otdf-python/src/otdf_python/sdk.py @@ -365,16 +365,24 @@ def is_tdf(data: bytes | BinaryIO) -> bool: bool: True if the data is a TDF, False otherwise """ + import json import zipfile from io import BytesIO + from otdf_python.tdf_reader import resolve_manifest_name, resolve_payload_name + try: file_like = BytesIO(data) if isinstance(data, bytes | bytearray) else data with zipfile.ZipFile(file_like) as zf: - names = set(zf.namelist()) - return {"0.manifest.json", "0.payload"}.issubset(names) and len( - names - ) == 2 + names = zf.namelist() + manifest_name = resolve_manifest_name(names) + manifest = json.loads(zf.read(manifest_name)) + payload = ( + manifest.get("payload") if isinstance(manifest, dict) else None + ) + url = payload.get("url") if isinstance(payload, dict) else None + resolve_payload_name(url, names) + return True except Exception: return False diff --git a/packages/otdf-python/src/otdf_python/tdf.py b/packages/otdf-python/src/otdf_python/tdf.py index 8ad487ce..5672a066 100644 --- a/packages/otdf-python/src/otdf_python/tdf.py +++ b/packages/otdf-python/src/otdf_python/tdf.py @@ -28,6 +28,7 @@ ManifestSegment, ) from otdf_python.policy_stub import NULL_POLICY_UUID +from otdf_python.tdf_reader import resolve_manifest_name, resolve_payload_name from otdf_python.tdf_writer import TDFWriter @@ -399,7 +400,7 @@ def create_tdf( ) payload_info = ManifestPayload( type="reference", # Changed from "file" to "reference" to match Java SDK - url="0.payload", + url=TDFWriter.TDF_PAYLOAD_FILE_NAME, protocol="zip", mimeType=config.mime_type, # Use MIME type from config isEncrypted=True, # Changed from is_encrypted to isEncrypted @@ -433,7 +434,8 @@ def load_tdf( tdf_bytes_io = io.BytesIO(tdf_data) if isinstance(tdf_data, bytes) else tdf_data with zipfile.ZipFile(tdf_bytes_io, "r") as z: - manifest_json = z.read("0.manifest.json").decode() + names = z.namelist() + manifest_json = z.read(resolve_manifest_name(names)).decode() manifest = Manifest.from_json(manifest_json) if not manifest.encryptionInformation: @@ -464,7 +466,8 @@ def load_tdf( segments = ( manifest.encryptionInformation.integrityInformation.segments ) # Changed field name - encrypted_payload = z.read("0.payload") + payload_url = manifest.payload.url if manifest.payload else None + encrypted_payload = z.read(resolve_payload_name(payload_url, names)) payload = self._decrypt_segments(aesgcm, segments, encrypted_payload) return TDFReader(payload=payload, manifest=manifest) @@ -487,7 +490,8 @@ def read_payload( from .asym_crypto import AsymDecryption with zipfile.ZipFile(io.BytesIO(tdf_bytes), "r") as z: - manifest_json = z.read("0.manifest.json").decode() + names = z.namelist() + manifest_json = z.read(resolve_manifest_name(names)).decode() manifest = Manifest.from_json(manifest_json) if not manifest.encryptionInformation: @@ -510,7 +514,8 @@ def read_payload( segments = ( manifest.encryptionInformation.integrityInformation.segments ) # Changed field names - encrypted_payload = z.read("0.payload") + payload_url = manifest.payload.url if manifest.payload else None + encrypted_payload = z.read(resolve_payload_name(payload_url, names)) offset = 0 for seg in segments: enc_len = seg.encryptedSegmentSize # Changed field name diff --git a/packages/otdf-python/src/otdf_python/tdf_reader.py b/packages/otdf-python/src/otdf_python/tdf_reader.py index 7c85ffb3..1c33af49 100644 --- a/packages/otdf-python/src/otdf_python/tdf_reader.py +++ b/packages/otdf-python/src/otdf_python/tdf_reader.py @@ -1,15 +1,71 @@ """TDFReader is responsible for reading and processing Trusted Data Format (TDF) files.""" +import json +from collections.abc import Iterable + from .manifest import Manifest from .policy_object import PolicyObject from .sdk_exceptions import SDKException from .zip_reader import ZipReader -# Constants from TDFWriter -TDF_MANIFEST_FILE_NAME = "0.manifest.json" +# Spec (opentdf/spec schema/OpenTDF/README.md): the manifest entry MUST be +# `manifest.json` at the archive root. `0.manifest.json` is what every SDK +# wrote before this change and is accepted forever on read. +TDF_MANIFEST_FILE_NAME = "manifest.json" +LEGACY_TDF_MANIFEST_FILE_NAME = "0.manifest.json" +# Payload entry name. Written by TDFWriter and into manifest.payload.url. +# On read this is only a fallback for manifests with an empty payload.url. TDF_PAYLOAD_FILE_NAME = "0.payload" +def resolve_manifest_name(names: Iterable[str]) -> str: + """Return the zip entry holding the manifest, spec name first.""" + name_set = set(names) + for candidate in (TDF_MANIFEST_FILE_NAME, LEGACY_TDF_MANIFEST_FILE_NAME): + if candidate in name_set: + return candidate + raise ValueError("tdf doesn't contain a manifest") + + +def _is_safe_entry_name(name: str) -> bool: + if not name or name.startswith("/") or "\\" in name: + return False + return ".." not in name.split("/") + + +def resolve_payload_name(payload_url: str | None, names: Iterable[str]) -> str: + """Return the zip entry holding the payload. + + Uses manifest.payload.url when present; falls back to `0.payload` only + when the url is empty or missing. + """ + name_set = set(names) + if payload_url: + if not _is_safe_entry_name(payload_url): + raise ValueError(f"unsafe payload url in manifest: {payload_url!r}") + if payload_url in name_set: + return payload_url + raise ValueError(f"tdf doesn't contain payload entry {payload_url!r}") + if TDF_PAYLOAD_FILE_NAME in name_set: + return TDF_PAYLOAD_FILE_NAME + raise ValueError("tdf doesn't contain a payload") + + +def payload_url_from_manifest_json(manifest_text: str) -> str | None: + """Extract payload.url without requiring a fully valid manifest.""" + try: + data = json.loads(manifest_text) + except (TypeError, ValueError): + return None + if not isinstance(data, dict): + return None + payload = data.get("payload") + if not isinstance(payload, dict): + return None + url = payload.get("url") + return url if isinstance(url, str) else None + + class TDFReader: """TDFReader is responsible for reading and processing Trusted Data Format (TDF) files. @@ -31,15 +87,16 @@ def __init__(self, tdf): try: self._zip_reader = ZipReader(tdf) namelist = self._zip_reader.namelist() - - if TDF_MANIFEST_FILE_NAME not in namelist: - raise ValueError("tdf doesn't contain a manifest") - if TDF_PAYLOAD_FILE_NAME not in namelist: - raise ValueError("tdf doesn't contain a payload") - - # Store the names for later use - self._manifest_name = TDF_MANIFEST_FILE_NAME - self._payload_name = TDF_PAYLOAD_FILE_NAME + self._manifest_name = resolve_manifest_name(namelist) + manifest_text = self._zip_reader.read(self._manifest_name).decode("utf-8") + payload_url = payload_url_from_manifest_json(manifest_text) + self._payload_name = resolve_payload_name(payload_url, namelist) + except UnicodeDecodeError as e: + # UnicodeDecodeError is a ValueError subclass, but it means the + # manifest entry is corrupt, not that the tdf is missing an + # entry (the resolvers' own ValueErrors). Wrap it like any + # other unexpected failure instead of letting it pass through. + raise SDKException("Error initializing TDFReader") from e except Exception as e: if isinstance(e, ValueError): raise diff --git a/packages/otdf-python/src/otdf_python/tdf_writer.py b/packages/otdf-python/src/otdf_python/tdf_writer.py index bef7cfd8..918760cb 100644 --- a/packages/otdf-python/src/otdf_python/tdf_writer.py +++ b/packages/otdf-python/src/otdf_python/tdf_writer.py @@ -8,8 +8,10 @@ class TDFWriter: """TDF file writer for creating encrypted TDF packages.""" + # Spec: manifest entry MUST be `manifest.json` at the archive root. + TDF_MANIFEST_FILE_NAME = "manifest.json" + # Payload entry name; TDF.create_tdf writes this same value into manifest.payload.url. TDF_PAYLOAD_FILE_NAME = "0.payload" - TDF_MANIFEST_FILE_NAME = "0.manifest.json" def __init__(self, out_stream: io.BytesIO | None = None): """Initialize TDF writer.""" diff --git a/tests/integration/test_cli_tdf_validation.py b/tests/integration/test_cli_tdf_validation.py index dd71ff3e..6029bb12 100644 --- a/tests/integration/test_cli_tdf_validation.py +++ b/tests/integration/test_cli_tdf_validation.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from otdf_python.tdf_reader import TDF_MANIFEST_FILE_NAME, TDF_PAYLOAD_FILE_NAME +from otdf_python.tdf_reader import resolve_manifest_name from tests.support_cli_args import ( run_cli_decrypt, @@ -22,6 +22,17 @@ run_otdfctl_encrypt_command, ) +# Accepted, documented interop break: released otdfctl still looks for the +# legacy `0.manifest.json` zip entry, while the Python SDK now writes the +# spec-mandated `manifest.json` entry (see README "TDF container format"). +# An upstream otdfctl/platform reader-fallback PR is planned; until it lands, +# any scenario where otdfctl decrypts Python-produced TDF output is expected +# to fail with "zip: file not found". +OTDFCTL_DECRYPT_XFAIL_REASON = ( + "otdfctl cannot read the spec manifest.json entry yet " + "(upstream reader-fallback pending)" +) + def _create_test_input_file(temp_path: Path, content: str) -> Path: """Create a test input file with the given content.""" @@ -89,18 +100,14 @@ def _validate_tdf_zip_structure(tdf_path: Path) -> None: f" {i + 1}. {filename} (size: {file_info.file_size} bytes, compressed: {file_info.compress_size} bytes)" ) - # TDF files should contain specific files - required_files = [TDF_MANIFEST_FILE_NAME, TDF_PAYLOAD_FILE_NAME] - for required_file in required_files: - assert required_file in file_list, ( - f"TDF missing required file: {required_file}" - ) + names = zip_file.namelist() + manifest_name = resolve_manifest_name(names) + manifest_content = zip_file.read(manifest_name) + manifest_data = json.loads(manifest_content) + assert manifest_data["payload"]["url"] in names # Validate manifest.json can be read and parsed try: - manifest_content = zip_file.read(TDF_MANIFEST_FILE_NAME) - manifest_data = json.loads(manifest_content.decode("utf-8")) - print("\n=== Manifest Structure Analysis ===") print(f"Manifest size: {len(manifest_content)} bytes") print(f"Top-level keys: {list(manifest_data.keys())}") @@ -233,8 +240,17 @@ def _run_otdfctl_decrypt( temp_path: Path, collect_server_logs, expected_content: str, + *, + expect_failure_reason: str | None = None, ) -> Path: - """Run otdfctl decrypt on a TDF file and verify the decrypted content matches expected.""" + """Run otdfctl decrypt on a TDF file and verify the decrypted content matches expected. + + If `expect_failure_reason` is set, a nonzero exit from this specific + otdfctl-decrypt step is treated as an accepted, documented interop break + (xfail) instead of a hard failure. Once otdfctl gains the ability to read + the spec `manifest.json` entry, this branch is simply never taken and the + test reports a normal pass. + """ decrypt_output = temp_path / f"{tdf_path.stem}_decrypted.txt" otdfctl_decrypt_result = run_otdfctl_decrypt_command( @@ -244,6 +260,14 @@ def _run_otdfctl_decrypt( cwd=temp_path, ) + if expect_failure_reason is not None and otdfctl_decrypt_result.returncode != 0: + print( + "otdfctl decrypt failed as expected (accepted interop break):\n" + f"stdout={otdfctl_decrypt_result.stdout}\n" + f"stderr={otdfctl_decrypt_result.stderr}" + ) + pytest.xfail(expect_failure_reason) + handle_subprocess_error( otdfctl_decrypt_result, collect_server_logs, "otdfctl decrypt" ) @@ -356,13 +380,22 @@ def test_python_encrypt(collect_server_logs, temp_credentials_file, project_root validate_tdf3_file(python_tdf_output, "Python CLI") _validate_tdf_zip_structure(python_tdf_output) + # Python writer must emit the spec-named manifest entry, not the legacy name. + with zipfile.ZipFile(python_tdf_output, "r") as zip_file: + python_names = zip_file.namelist() + assert "manifest.json" in python_names + assert "0.manifest.json" not in python_names + # Test that the TDF can be decrypted by otdfctl + # Accepted, documented interop break: otdfctl still looks for the + # legacy `0.manifest.json` entry (see README "TDF container format"). _run_otdfctl_decrypt( python_tdf_output, temp_credentials_file, temp_path, collect_server_logs, input_content, + expect_failure_reason=OTDFCTL_DECRYPT_XFAIL_REASON, ) print( @@ -430,12 +463,15 @@ def test_cross_tool_compatibility( ) # Decrypt with otdfctl + # Accepted, documented interop break: otdfctl still looks for the + # legacy `0.manifest.json` entry (see README "TDF container format"). _run_otdfctl_decrypt( python_tdf_output, temp_credentials_file, temp_path, collect_server_logs, input_content, + expect_failure_reason=OTDFCTL_DECRYPT_XFAIL_REASON, ) print( @@ -488,12 +524,15 @@ def test_different_content_types( validate_tdf3_file(python_tdf_output, f"Python CLI ({filename})") # Decrypt and validate content + # Accepted, documented interop break: otdfctl still looks for the + # legacy `0.manifest.json` entry (see README "TDF container format"). _run_otdfctl_decrypt( python_tdf_output, temp_credentials_file, temp_path, collect_server_logs, content, + expect_failure_reason=OTDFCTL_DECRYPT_XFAIL_REASON, ) print(f"✓ Successfully processed {filename}") diff --git a/tests/test_cli_supports.py b/tests/test_cli_supports.py index 660400a8..963ba4b0 100644 --- a/tests/test_cli_supports.py +++ b/tests/test_cli_supports.py @@ -42,3 +42,7 @@ def test_supports_unknown_exit_2(): check=False, ) assert r.returncode == 2 + + +def test_supports_spec_container(): + assert cmd_supports(SimpleNamespace(feature="spec-container")) == 0 diff --git a/tests/test_manifest_format.py b/tests/test_manifest_format.py index 3eed648d..b35ccf3f 100644 --- a/tests/test_manifest_format.py +++ b/tests/test_manifest_format.py @@ -3,6 +3,7 @@ import json from otdf_python.config import KASInfo, TDFConfig +from otdf_python.manifest import Manifest from otdf_python.tdf import TDF from tests.mock_crypto import generate_rsa_keypair @@ -99,3 +100,48 @@ def test_manifest_roundtrip_serialization(): "wrappedKey" ] assert original_wrapped_key == roundtrip_wrapped_key + + +def _minimal(**top): + base = { + "payload": { + "type": "reference", + "url": "0.payload", + "protocol": "zip", + "mimeType": "text/plain", + "isEncrypted": True, + } + } + base.update(top) + return base + + +def test_spec_version_prefers_schema_version(): + m = _minimal(schemaVersion="4.3.0", tdf_spec_version="9.9.9") + m["payload"]["tdf_spec_version"] = "8.8.8" + assert Manifest.from_json(json.dumps(m)).spec_version() == "4.3.0" + + +def test_spec_version_falls_back_to_top_level_tdf_spec_version(): + m = _minimal(tdf_spec_version="9.9.9") + m["payload"]["tdf_spec_version"] = "8.8.8" + assert Manifest.from_json(json.dumps(m)).spec_version() == "9.9.9" + + +def test_spec_version_falls_back_to_payload_tdf_spec_version(): + m = _minimal() + m["payload"]["tdf_spec_version"] = "8.8.8" + assert Manifest.from_json(json.dumps(m)).spec_version() == "8.8.8" + + +def test_spec_version_absent_is_none(): + assert Manifest.from_json(json.dumps(_minimal())).spec_version() is None + + +def test_tdf_spec_version_never_serialized(): + m = _minimal(schemaVersion="4.3.0", tdf_spec_version="9.9.9") + m["payload"]["tdf_spec_version"] = "8.8.8" + out = json.loads(Manifest.from_json(json.dumps(m)).to_json()) + assert "tdf_spec_version" not in out + assert "tdf_spec_version" not in out["payload"] + assert out["schemaVersion"] == "4.3.0" diff --git a/tests/test_sdk.py b/tests/test_sdk.py index 86f51c16..d4a395db 100644 --- a/tests/test_sdk.py +++ b/tests/test_sdk.py @@ -1,5 +1,9 @@ """Basic tests for the Python SDK class.""" +import io +import json +import zipfile + import pytest from otdf_python.sdk import SDK @@ -105,3 +109,33 @@ def test_assertion_exception(): with pytest.raises(SDK.AssertionException, match="assertion failed"): raise SDK.AssertionException("assertion failed", "id123") + + +def _zip_with(entries: dict[str, bytes]) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + for name, content in entries.items(): + z.writestr(name, content) + return buf.getvalue() + + +def test_is_tdf_accepts_spec_layout(): + manifest = json.dumps({"payload": {"url": "0.payload"}}).encode() + assert SDK.is_tdf(_zip_with({"manifest.json": manifest, "0.payload": b"x"})) + + +def test_is_tdf_accepts_legacy_layout(): + manifest = json.dumps({"payload": {"url": "0.payload"}}).encode() + assert SDK.is_tdf(_zip_with({"0.manifest.json": manifest, "0.payload": b"x"})) + + +def test_is_tdf_accepts_extra_entries_and_custom_payload_name(): + manifest = json.dumps({"payload": {"url": "blob"}}).encode() + assert SDK.is_tdf( + _zip_with({"manifest.json": manifest, "blob": b"x", "extra.txt": b"y"}) + ) + + +def test_is_tdf_rejects_missing_payload_entry(): + manifest = json.dumps({"payload": {"url": "blob"}}).encode() + assert not SDK.is_tdf(_zip_with({"manifest.json": manifest, "0.payload": b"x"})) diff --git a/tests/test_tdf.py b/tests/test_tdf.py index a12c4784..e15e95c7 100644 --- a/tests/test_tdf.py +++ b/tests/test_tdf.py @@ -28,11 +28,14 @@ def test_tdf_create_and_load(): data = out.getvalue() if hasattr(out, "getvalue") else out.read() with zipfile.ZipFile(io.BytesIO(data), "r") as z: files = z.namelist() - assert "0.manifest.json" in files - assert "0.payload" in files - manifest_json = json.loads(z.read("0.manifest.json").decode()) + assert "manifest.json" in files + assert "0.manifest.json" not in files + manifest_json = json.loads(z.read("manifest.json").decode()) assert manifest_json["schemaVersion"] == TDF.TDF_VERSION - encrypted_payload = z.read("0.payload") + assert "tdf_spec_version" not in manifest_json + assert "tdf_spec_version" not in manifest_json["payload"] + assert manifest_json["payload"]["url"] in files + encrypted_payload = z.read(manifest_json["payload"]["url"]) assert encrypted_payload != payload # Should be encrypted assert len(encrypted_payload) > 0 # Test round-trip decryption @@ -60,3 +63,72 @@ def test_tdf_multi_kas_roundtrip(): reader_config = TDFReaderConfig(kas_private_key=priv) dec = tdf.load_tdf(data, reader_config) assert dec.payload == payload + + +def _make_tdf_bytes(tdf, payload, config): + _, _, out = tdf.create_tdf(payload, config) + return out.getvalue() if hasattr(out, "getvalue") else out.read() + + +def _rewrite_zip(data: bytes, rename: dict[str, str], manifest_edit=None) -> bytes: + """Copy a zip, renaming entries and optionally editing the manifest JSON.""" + src = zipfile.ZipFile(io.BytesIO(data)) + dst_buf = io.BytesIO() + with zipfile.ZipFile(dst_buf, "w") as dst: + for name in src.namelist(): + content = src.read(name) + if name.endswith("manifest.json") and manifest_edit: + m = json.loads(content) + manifest_edit(m) + content = json.dumps(m).encode() + dst.writestr(rename.get(name, name), content) + return dst_buf.getvalue() + + +def test_load_tdf_reads_legacy_manifest_name(): + tdf = TDF() + kas_private_key, kas_public_key = generate_rsa_keypair() + kas_info = KASInfo( + url="https://kas.example.com", public_key=kas_public_key, kid="k" + ) + config = TDFConfig(kas_info_list=[kas_info], tdf_private_key=kas_private_key) + data = _make_tdf_bytes(tdf, b"legacy", config) + legacy = _rewrite_zip(data, {"manifest.json": "0.manifest.json"}) + with zipfile.ZipFile(io.BytesIO(legacy)) as z: + assert "0.manifest.json" in z.namelist() + decrypted = tdf.load_tdf(legacy, TDFReaderConfig(kas_private_key=kas_private_key)) + assert decrypted.payload == b"legacy" + + +def test_load_tdf_locates_payload_by_manifest_url(): + tdf = TDF() + kas_private_key, kas_public_key = generate_rsa_keypair() + kas_info = KASInfo( + url="https://kas.example.com", public_key=kas_public_key, kid="k" + ) + config = TDFConfig(kas_info_list=[kas_info], tdf_private_key=kas_private_key) + data = _make_tdf_bytes(tdf, b"renamed", config) + + def set_url(m): + m["payload"]["url"] = "data.bin" + + renamed = _rewrite_zip(data, {"0.payload": "data.bin"}, manifest_edit=set_url) + decrypted = tdf.load_tdf(renamed, TDFReaderConfig(kas_private_key=kas_private_key)) + assert decrypted.payload == b"renamed" + + +def test_load_tdf_rejects_unsafe_payload_url(): + tdf = TDF() + kas_private_key, kas_public_key = generate_rsa_keypair() + kas_info = KASInfo( + url="https://kas.example.com", public_key=kas_public_key, kid="k" + ) + config = TDFConfig(kas_info_list=[kas_info], tdf_private_key=kas_private_key) + data = _make_tdf_bytes(tdf, b"x", config) + + def set_url(m): + m["payload"]["url"] = "../0.payload" + + bad = _rewrite_zip(data, {}, manifest_edit=set_url) + with pytest.raises(ValueError, match="unsafe"): + tdf.load_tdf(bad, TDFReaderConfig(kas_private_key=kas_private_key)) diff --git a/tests/test_tdf_key_management.py b/tests/test_tdf_key_management.py index 4f749d34..d69257cf 100644 --- a/tests/test_tdf_key_management.py +++ b/tests/test_tdf_key_management.py @@ -90,7 +90,7 @@ def _create_mock_tdf(self): ) # Add manifest to zip - zf.writestr("0.manifest.json", manifest.to_json()) + zf.writestr("manifest.json", manifest.to_json()) # Add encrypted payload zf.writestr( diff --git a/tests/test_tdf_reader.py b/tests/test_tdf_reader.py index 40a2d8ee..ab80b030 100644 --- a/tests/test_tdf_reader.py +++ b/tests/test_tdf_reader.py @@ -2,14 +2,20 @@ import io import json +import re from unittest.mock import MagicMock, patch import pytest from otdf_python.policy_object import PolicyObject +from otdf_python.sdk_exceptions import SDKException from otdf_python.tdf_reader import ( + LEGACY_TDF_MANIFEST_FILE_NAME, TDF_MANIFEST_FILE_NAME, TDF_PAYLOAD_FILE_NAME, TDFReader, + payload_url_from_manifest_json, + resolve_manifest_name, + resolve_payload_name, ) @@ -150,3 +156,92 @@ def test_read_policy_object(self, mock_manifest, mock_zip_reader): assert result.body.dissem == ["user1", "user2"] mock_reader.read.assert_called_with(TDF_MANIFEST_FILE_NAME) mock_manifest.from_json.assert_called_once() + + +class TestEntryNameResolvers: + def test_manifest_prefers_spec_name(self): + names = ["0.manifest.json", "manifest.json", "0.payload"] + assert resolve_manifest_name(names) == "manifest.json" + + def test_manifest_falls_back_to_legacy_name(self): + assert ( + resolve_manifest_name(["0.manifest.json", "0.payload"]) == "0.manifest.json" + ) + + def test_manifest_missing(self): + with pytest.raises(ValueError, match="tdf doesn't contain a manifest"): + resolve_manifest_name(["0.payload"]) + + def test_payload_from_url(self): + assert ( + resolve_payload_name("data.bin", ["manifest.json", "data.bin"]) + == "data.bin" + ) + + def test_payload_url_missing_entry_is_error_and_quotes_url(self): + with pytest.raises(ValueError, match=re.escape("'data.bin'")): + resolve_payload_name("data.bin", ["manifest.json", "0.payload"]) + + def test_payload_fallback_when_url_empty(self): + assert resolve_payload_name("", ["manifest.json", "0.payload"]) == "0.payload" + assert resolve_payload_name(None, ["manifest.json", "0.payload"]) == "0.payload" + + def test_payload_fallback_missing(self): + with pytest.raises(ValueError, match="tdf doesn't contain a payload"): + resolve_payload_name(None, ["manifest.json"]) + + @pytest.mark.parametrize("bad", ["../x", "/abs", "a\\b", "x/../y"]) + def test_payload_unsafe_url_rejected(self, bad): + with pytest.raises(ValueError, match="unsafe"): + resolve_payload_name(bad, ["manifest.json", bad]) + + def test_payload_url_from_manifest_json(self): + assert ( + payload_url_from_manifest_json('{"payload": {"url": "p.bin"}}') == "p.bin" + ) + assert payload_url_from_manifest_json('{"payload": {}}') is None + assert payload_url_from_manifest_json("{}") is None + assert payload_url_from_manifest_json("not json") is None + + +class TestTDFReaderEntryResolution: + def _reader_with(self, names, manifest_text): + with patch("otdf_python.tdf_reader.ZipReader") as mock_zip_reader: + inst = mock_zip_reader.return_value + inst.namelist.return_value = names + inst.read.side_effect = lambda n: ( + manifest_text.encode() if n.endswith("manifest.json") else b"PAYLOAD" + ) + reader = TDFReader(io.BytesIO(b"x")) + return reader, inst + + def test_reads_legacy_manifest_name(self): + reader, inst = self._reader_with( + ["0.manifest.json", "0.payload"], '{"payload": {"url": "0.payload"}}' + ) + assert reader.manifest() == '{"payload": {"url": "0.payload"}}' + inst.read.assert_called_with(LEGACY_TDF_MANIFEST_FILE_NAME) + + def test_payload_name_comes_from_manifest_url(self): + reader, inst = self._reader_with( + ["manifest.json", "custom.bin"], '{"payload": {"url": "custom.bin"}}' + ) + buf = bytearray(7) + assert reader.read_payload_bytes(buf) == 7 + inst.read.assert_called_with("custom.bin") + + def test_payload_url_missing_entry_fails_at_init(self): + with pytest.raises(ValueError, match=re.escape("'custom.bin'")): + self._reader_with( + ["manifest.json", "0.payload"], '{"payload": {"url": "custom.bin"}}' + ) + + def test_invalid_utf8_manifest_wrapped_as_sdk_exception(self): + with patch("otdf_python.tdf_reader.ZipReader") as mock_zip_reader: + inst = mock_zip_reader.return_value + inst.namelist.return_value = ["manifest.json", "0.payload"] + inst.read.side_effect = lambda n: ( + b"\xff\xfe" if n == "manifest.json" else b"PAYLOAD" + ) + with pytest.raises(SDKException): + TDFReader(io.BytesIO(b"x")) diff --git a/tests/test_tdf_writer.py b/tests/test_tdf_writer.py index 354045a6..7f1998ea 100644 --- a/tests/test_tdf_writer.py +++ b/tests/test_tdf_writer.py @@ -22,7 +22,7 @@ def test_append_manifest_and_payload(self): self.assertGreater(size, 0) out.seek(0) with zipfile.ZipFile(out, "r") as z: - self.assertEqual(z.read("0.manifest.json"), manifest.encode("utf-8")) + self.assertEqual(z.read("manifest.json"), manifest.encode("utf-8")) self.assertEqual(z.read("0.payload"), b"payload data") def test_getvalue(self): @@ -34,7 +34,7 @@ def test_getvalue(self): writer.finish() data = writer.getvalue() with zipfile.ZipFile(io.BytesIO(data), "r") as z: - self.assertEqual(z.read("0.manifest.json"), b"{}") + self.assertEqual(z.read("manifest.json"), b"{}") self.assertEqual(z.read("0.payload"), b"abc") def test_large_payload_chunks(self): @@ -63,6 +63,16 @@ def test_error_on_write_after_finish(self): with self.assertRaises(ValueError), writer.payload() as f: f.write(b"should fail") + def test_manifest_entry_is_spec_name(self): + writer = TDFWriter() + writer.append_manifest("{}") + with writer.payload() as f: + f.write(b"x") + writer.finish() + with zipfile.ZipFile(io.BytesIO(writer.getvalue()), "r") as z: + self.assertEqual(sorted(z.namelist()), ["0.payload", "manifest.json"]) + self.assertNotIn("0.manifest.json", z.namelist()) + if __name__ == "__main__": unittest.main()