From d912ada180fcf881c25c4db2452fa8ca09de42e8 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:36:36 -0400 Subject: [PATCH 1/7] perf(pdf): bound OCR upload and rendered-page memory --- CHANGELOG.md | 6 + .../performance/benchmark_pdf_ocr_memory.py | 42 +++++ .../pdf_image/test_pdf_streaming_memory.py | 164 ++++++++++++++++++ unstructured/__version__.py | 2 +- unstructured/partition/pdf.py | 21 ++- unstructured/partition/pdf_image/ocr.py | 8 +- .../partition/pdf_image/pdf_image_utils.py | 79 +++++---- 7 files changed, 280 insertions(+), 42 deletions(-) create mode 100644 scripts/performance/benchmark_pdf_ocr_memory.py create mode 100644 test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 75720b6c14..053807b4a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.12 + +### Fixes + +- **Bound PDF OCR upload and render memory.** Copy uploaded OCR inputs to temporary files in 1 MiB blocks, render OCR-only PDFs one page at a time, and close owned page images after OCR. Preserve native rendered pixels, image metadata, output order and caller-owned upload streams. Large uploaded PDFs now use temporary disk instead of a full-size in-memory copy; retained output still scales with extracted content. + ## 0.27.11 ### Fixes diff --git a/scripts/performance/benchmark_pdf_ocr_memory.py b/scripts/performance/benchmark_pdf_ocr_memory.py new file mode 100644 index 0000000000..735cff0cb1 --- /dev/null +++ b/scripts/performance/benchmark_pdf_ocr_memory.py @@ -0,0 +1,42 @@ +"""Exercise the real OCR-only PDF partitioner with deterministic blank pages.""" + +import argparse +import hashlib +import json +import resource +import tempfile +import time +from pathlib import Path + +from pypdf import PdfWriter + +from unstructured.partition.pdf import partition_pdf + +parser = argparse.ArgumentParser() +parser.add_argument("--pages", type=int, default=20) +parser.add_argument("--page-points", type=int, default=1200) +args = parser.parse_args() +with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "blank-pages.pdf" + writer = PdfWriter() + for _ in range(args.pages): + writer.add_blank_page(width=args.page_points, height=args.page_points) + with path.open("wb") as destination: + writer.write(destination) + print(json.dumps({"phase": "start", **vars(args)}), flush=True) + started = time.perf_counter() + with path.open("rb") as source: + elements = partition_pdf(file=source, strategy="ocr_only", languages=["eng"]) + output = json.dumps([element.to_dict() for element in elements], sort_keys=True).encode() + print( + json.dumps( + { + "phase": "complete", + "elapsed_seconds": time.perf_counter() - started, + "peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "output_count": len(elements), + "sha256": hashlib.sha256(output).hexdigest(), + } + ), + flush=True, + ) diff --git a/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py b/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py new file mode 100644 index 0000000000..d983e3539f --- /dev/null +++ b/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py @@ -0,0 +1,164 @@ +"""Preserve upload ownership and native render output while bounding working memory.""" + +import io +import os +import tempfile +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import Mock + +import pytest +from PIL import Image + +from unstructured.partition.pdf_image import ocr, pdf_image_utils + + +class BoundedBytesIO(io.BytesIO): + def read(self, size=-1): + assert 0 < size <= 1024 * 1024 + return super().read(size) + + +@pytest.mark.parametrize("kind", ["bytes", "bytesio", "spool-memory", "spool-disk", "buffered"]) +def test_uploaded_pdf_is_copied_in_bounded_reads_with_original_cursor_semantics(kind, tmp_path): + stack = ExitStack() + payload = b"document" * 300000 + path = tmp_path / "source.pdf" + path.write_bytes(payload) + if kind == "bytes": + source = payload + elif kind == "bytesio": + source = BoundedBytesIO(payload) + elif kind == "buffered": + source = stack.enter_context(path.open("rb")) + else: + source = stack.enter_context( + tempfile.SpooledTemporaryFile( # noqa: SIM115 - owned by ExitStack + max_size=1 if kind == "spool-disk" else len(payload) + 1 + ) + ) + source.write(payload) + if kind != "bytes": + source.seek(3) + with pdf_image_utils._pdf_source_path("", source) as copied_path: + assert Path(copied_path).read_bytes() == payload + assert kind == "bytes" or not source.closed + assert not os.path.exists(copied_path) + if kind != "bytes": + assert source.tell() == (0 if kind.startswith("spool") else 3) + stack.close() + + +def test_render_generator_does_not_materialize_a_decoded_page_batch(monkeypatch): + monkeypatch.setattr( + pdf_image_utils.pdf2image, "pdfinfo_from_path", lambda *a, **k: {"Pages": 10} + ) + calls = [] + + def render(**kwargs): + calls.append((kwargs["first_page"], kwargs["last_page"])) + assert kwargs["first_page"] == kwargs["last_page"] + return [Image.new("RGB", (2, 2), (kwargs["first_page"], 0, 0))] + + monkeypatch.setattr(pdf_image_utils, "render_pdf_to_image", render) + pages = pdf_image_utils.convert_pdf_to_images(filename="input.pdf") + first = next(pages) + assert calls == [(1, 1)] + rest = list(pages) + assert calls == [(page, page) for page in range(1, 11)] + assert first.getpixel((0, 0)) == (1, 0, 0) + assert rest[-1].getpixel((0, 0)) == (10, 0, 0) + for image in [first, *rest]: + image.close() + + +def test_generator_close_removes_owned_upload_without_closing_borrowed_stream(monkeypatch): + paths = [] + + def info(path, **kwargs): + paths.append(path) + return {"Pages": 2} + + monkeypatch.setattr(pdf_image_utils.pdf2image, "pdfinfo_from_path", info) + monkeypatch.setattr( + pdf_image_utils, "render_pdf_to_image", lambda **k: [Image.new("RGB", (1, 1))] + ) + source = BoundedBytesIO(b"pdf bytes") + pages = pdf_image_utils.convert_pdf_to_images(file=source) + image = next(pages) + assert os.path.exists(paths[0]) + pages.close() + assert not os.path.exists(paths[0]) + assert not source.closed + assert image.getpixel((0, 0)) == (0, 0, 0) + image.close() + + +def test_process_data_with_ocr_copies_only_remaining_stream_without_retaining_a_full_read( + monkeypatch, +): + source = BoundedBytesIO(b"prefix" + b"document" * 300000) + source.seek(6) + paths = [] + result = object() + + def process(filename, **kwargs): + paths.append(filename) + assert Path(filename).read_bytes() == b"document" * 300000 + return result + + monkeypatch.setattr(ocr, "process_file_with_ocr", process) + assert ocr.process_data_with_ocr(source, Mock(), []) is result + assert not source.closed + assert source.tell() == len(source.getvalue()) + assert not os.path.exists(paths[0]) + + +@pytest.mark.parametrize("fail", [False, True]) +def test_ocr_only_closes_owned_render_images_on_success_and_failure(monkeypatch, fail): + from unstructured.partition import pdf + + image = Image.new("RGB", (2, 2)) + monkeypatch.setattr(pdf_image_utils, "convert_pdf_to_images", lambda *a, **k: iter([image])) + + def partition(**kwargs): + assert kwargs["image"].getpixel((0, 0)) == (0, 0, 0) + if fail: + raise RuntimeError("OCR failed") + return [] + + monkeypatch.setattr(pdf, "_partition_pdf_or_image_with_ocr_from_image", partition) + if fail: + with pytest.raises(RuntimeError, match="OCR failed"): + pdf._partition_pdf_or_image_with_ocr(filename="input.pdf") + else: + assert pdf._partition_pdf_or_image_with_ocr(filename="input.pdf") == [] + with pytest.raises(ValueError, match="closed image"): + image.getpixel((0, 0)) + + +@pytest.mark.parametrize( + "fixture", + ["rotated-page-90.pdf", "pdf/layout-parser-paper-fast.pdf", "pdf/embedded-images.pdf"], +) +def test_single_page_render_matches_native_batch_pixels_and_metadata(fixture): + from test_unstructured.unit_utils import example_doc_path + + path = str(example_doc_path(fixture)) + expected = pdf_image_utils.render_pdf_to_image( + filename=path, + dpi=pdf_image_utils.env_config.PDF_RENDER_DPI, + pdf_render_max_pixels_per_page=pdf_image_utils.env_config.PDF_RENDER_MAX_PIXELS_PER_PAGE, + ) + actual = list(pdf_image_utils.convert_pdf_to_images(filename=path)) + assert len(actual) == len(expected) + for baseline, candidate in zip(expected, actual): + assert (candidate.size, candidate.mode, candidate.format, candidate.info) == ( + baseline.size, + baseline.mode, + baseline.format, + baseline.info, + ) + assert candidate.tobytes() == baseline.tobytes() + baseline.close() + candidate.close() diff --git a/unstructured/__version__.py b/unstructured/__version__.py index bef9bb955e..f1527a87a7 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.11" # pragma: no cover +__version__ = "0.27.12" # pragma: no cover diff --git a/unstructured/partition/pdf.py b/unstructured/partition/pdf.py index 18fe1b7c0b..afb35cea50 100644 --- a/unstructured/partition/pdf.py +++ b/unstructured/partition/pdf.py @@ -1303,15 +1303,18 @@ def _partition_pdf_or_image_with_ocr( for page_number, image in enumerate( convert_pdf_to_images(filename, file, password=password), start=starting_page_number ): - page_elements = _partition_pdf_or_image_with_ocr_from_image( - image=image, - languages=languages, - ocr_languages=ocr_languages, - page_number=page_number, - include_page_breaks=include_page_breaks, - metadata_last_modified=metadata_last_modified, - **kwargs, - ) + try: + page_elements = _partition_pdf_or_image_with_ocr_from_image( + image=image, + languages=languages, + ocr_languages=ocr_languages, + page_number=page_number, + include_page_breaks=include_page_breaks, + metadata_last_modified=metadata_last_modified, + **kwargs, + ) + finally: + image.close() elements.extend(page_elements) return elements diff --git a/unstructured/partition/pdf_image/ocr.py b/unstructured/partition/pdf_image/ocr.py index 6fa89e4e94..550aff4c3c 100644 --- a/unstructured/partition/pdf_image/ocr.py +++ b/unstructured/partition/pdf_image/ocr.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shutil import tempfile from typing import IO, TYPE_CHECKING, Any, List, Optional, cast @@ -77,12 +78,13 @@ def process_data_with_ocr( Returns: DocumentLayout: The merged layout information obtained after OCR processing. """ - data_bytes = data if isinstance(data, bytes) else data.read() - with tempfile.TemporaryDirectory() as tmp_dir_path: tmp_file_path = os.path.join(tmp_dir_path, "tmp_file") with open(tmp_file_path, "wb") as tmp_file: - tmp_file.write(data_bytes) + if isinstance(data, bytes): + tmp_file.write(data) + else: + shutil.copyfileobj(data, tmp_file, length=1024 * 1024) merged_layouts = process_file_with_ocr( filename=tmp_file_path, diff --git a/unstructured/partition/pdf_image/pdf_image_utils.py b/unstructured/partition/pdf_image/pdf_image_utils.py index b81714a9f7..e7df5aec94 100644 --- a/unstructured/partition/pdf_image/pdf_image_utils.py +++ b/unstructured/partition/pdf_image/pdf_image_utils.py @@ -3,10 +3,12 @@ import base64 import os import re +import shutil import tempfile import unicodedata +from contextlib import contextmanager from copy import deepcopy -from io import BytesIO +from io import BufferedReader, BytesIO, TextIOWrapper from pathlib import Path, PurePath from typing import IO, TYPE_CHECKING, BinaryIO, Iterator, List, Optional, Tuple, Union, cast @@ -20,7 +22,7 @@ from unstructured.documents.elements import ElementType from unstructured.errors import UnprocessableEntityError from unstructured.logger import logger -from unstructured.partition.common.common import convert_to_bytes, exactly_one +from unstructured.partition.common.common import exactly_one from unstructured.partition.utils.config import env_config if TYPE_CHECKING: @@ -399,34 +401,53 @@ def convert_pdf_to_images( chunk_size: int = 10, password: Optional[str] = None, ) -> Iterator[Image.Image]: - # Convert a PDF in small chunks of pages at a time (e.g. 1-10, 11-20... and so on) + """Yield owned page images without retaining the upload or a batch of decoded pages.""" exactly_one(filename=filename, file=file) - if file is not None: - f_bytes = convert_to_bytes(file) - info = pdf2image.pdfinfo_from_bytes(f_bytes, userpw=password) - else: - f_bytes = None - info = pdf2image.pdfinfo_from_path(filename, userpw=password) - - total_pages = info["Pages"] - for start_page in range(1, total_pages + 1, chunk_size): - end_page = min(start_page + chunk_size - 1, total_pages) - try: - chunk_images = render_pdf_to_image( - filename=filename if f_bytes is None else None, - file=f_bytes, - dpi=env_config.PDF_RENDER_DPI, - first_page=start_page, - last_page=end_page, - password=password, - pdf_render_max_pixels_per_page=env_config.PDF_RENDER_MAX_PIXELS_PER_PAGE, - ) - except PdfRenderTooLargeError as exc: - raise UnprocessableEntityError(str(exc)) from exc - chunk_images = cast(List[Image.Image], chunk_images) - - for image in chunk_images: - yield image + with _pdf_source_path(filename, file) as source_path: + total_pages = pdf2image.pdfinfo_from_path(source_path, userpw=password)["Pages"] + for start_page in range(1, total_pages + 1, chunk_size): + end_page = min(start_page + chunk_size - 1, total_pages) + for page_number in range(start_page, end_page + 1): + try: + images = render_pdf_to_image( + filename=source_path, + dpi=env_config.PDF_RENDER_DPI, + first_page=page_number, + last_page=page_number, + password=password, + pdf_render_max_pixels_per_page=env_config.PDF_RENDER_MAX_PIXELS_PER_PAGE, + ) + except PdfRenderTooLargeError as exc: + raise UnprocessableEntityError(str(exc)) from exc + # The caller owns the yielded image and may retain it after iteration. + yield from cast(List[Image.Image], images) + del images + + +@contextmanager +def _pdf_source_path(filename: str, file: Optional[bytes | IO[bytes]]) -> Iterator[str]: + """Materialize uploaded PDFs on disk with the existing stream-position semantics.""" + if file is None: + yield filename + return + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "upload.pdf") + with open(path, "wb") as destination: + if isinstance(file, bytes): + destination.write(file) + elif isinstance(file, (BytesIO, tempfile.SpooledTemporaryFile)): + position = 0 if isinstance(file, tempfile.SpooledTemporaryFile) else file.tell() + try: + file.seek(0) + shutil.copyfileobj(file, destination, length=1024 * 1024) + finally: + file.seek(position) + elif isinstance(file, (TextIOWrapper, BufferedReader)): + with open(file.name, "rb") as source: + shutil.copyfileobj(source, destination, length=1024 * 1024) + else: + raise ValueError("Invalid file-like object type") + yield path def remove_control_characters(text: str) -> str: From 8fe306866e0c54ead20b07df62fd2e059b2fd890 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:42:59 -0400 Subject: [PATCH 2/7] fix(benchmark): normalize peak RSS units on macOS --- scripts/performance/benchmark_pdf_ocr_memory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/performance/benchmark_pdf_ocr_memory.py b/scripts/performance/benchmark_pdf_ocr_memory.py index 735cff0cb1..cd0b56c6e2 100644 --- a/scripts/performance/benchmark_pdf_ocr_memory.py +++ b/scripts/performance/benchmark_pdf_ocr_memory.py @@ -4,6 +4,7 @@ import hashlib import json import resource +import sys import tempfile import time from pathlib import Path @@ -33,7 +34,8 @@ { "phase": "complete", "elapsed_seconds": time.perf_counter() - started, - "peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + // (1024 if sys.platform == "darwin" else 1), "output_count": len(elements), "sha256": hashlib.sha256(output).hexdigest(), } From 0ab5c595c57964b05911c93bfd3e9eca019c3a28 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:23:18 -0400 Subject: [PATCH 3/7] chore: retarget release to 0.27.13 --- CHANGELOG.md | 2 +- unstructured/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 053807b4a7..43ef95fa9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 0.27.12 +## 0.27.13 ### Fixes diff --git a/unstructured/__version__.py b/unstructured/__version__.py index f1527a87a7..5cdb02ca0c 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.12" # pragma: no cover +__version__ = "0.27.13" # pragma: no cover From 2a8a40155d1164d639ff71f694368d949aae323c Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:25:18 -0400 Subject: [PATCH 4/7] refactor(pdf): iterate pages directly in convert_pdf_to_images --- .../partition/pdf_image/pdf_image_utils.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/unstructured/partition/pdf_image/pdf_image_utils.py b/unstructured/partition/pdf_image/pdf_image_utils.py index e7df5aec94..800d170d2b 100644 --- a/unstructured/partition/pdf_image/pdf_image_utils.py +++ b/unstructured/partition/pdf_image/pdf_image_utils.py @@ -405,28 +405,29 @@ def convert_pdf_to_images( exactly_one(filename=filename, file=file) with _pdf_source_path(filename, file) as source_path: total_pages = pdf2image.pdfinfo_from_path(source_path, userpw=password)["Pages"] - for start_page in range(1, total_pages + 1, chunk_size): - end_page = min(start_page + chunk_size - 1, total_pages) - for page_number in range(start_page, end_page + 1): - try: - images = render_pdf_to_image( - filename=source_path, - dpi=env_config.PDF_RENDER_DPI, - first_page=page_number, - last_page=page_number, - password=password, - pdf_render_max_pixels_per_page=env_config.PDF_RENDER_MAX_PIXELS_PER_PAGE, - ) - except PdfRenderTooLargeError as exc: - raise UnprocessableEntityError(str(exc)) from exc - # The caller owns the yielded image and may retain it after iteration. - yield from cast(List[Image.Image], images) - del images + for page_number in range(1, total_pages + 1): + try: + images = render_pdf_to_image( + filename=source_path, + dpi=env_config.PDF_RENDER_DPI, + first_page=page_number, + last_page=page_number, + password=password, + pdf_render_max_pixels_per_page=env_config.PDF_RENDER_MAX_PIXELS_PER_PAGE, + ) + except PdfRenderTooLargeError as exc: + raise UnprocessableEntityError(str(exc)) from exc + # The caller owns the yielded image and may retain it after iteration. + yield from cast(List[Image.Image], images) + del images @contextmanager def _pdf_source_path(filename: str, file: Optional[bytes | IO[bytes]]) -> Iterator[str]: - """Materialize uploaded PDFs on disk with the existing stream-position semantics.""" + """Yield a filesystem path for `filename` or a temporary on-disk copy of `file`. + + A `BytesIO` cursor is restored after the copy and a `SpooledTemporaryFile` is left rewound. + """ if file is None: yield filename return From b3b09cb1b172326288b990fd9cd10a27e158fe47 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:02:27 -0400 Subject: [PATCH 5/7] fix(pdf): accept any readable, seekable stream for OCR-only uploads --- CHANGELOG.md | 2 +- .../pdf_image/test_pdf_streaming_memory.py | 36 +++++++++++++++++++ .../partition/pdf_image/pdf_image_utils.py | 13 +++---- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43ef95fa9e..3fbcce1a40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Fixes -- **Bound PDF OCR upload and render memory.** Copy uploaded OCR inputs to temporary files in 1 MiB blocks, render OCR-only PDFs one page at a time, and close owned page images after OCR. Preserve native rendered pixels, image metadata, output order and caller-owned upload streams. Large uploaded PDFs now use temporary disk instead of a full-size in-memory copy; retained output still scales with extracted content. +- **Bound PDF OCR upload and render memory.** Copy uploaded OCR inputs to temporary files in 1 MiB blocks, render OCR-only PDFs one page at a time, and close owned page images after OCR. Uploads may be any readable, seekable stream. Preserve native rendered pixels, image metadata, output order and caller-owned upload streams. Large uploaded PDFs now use temporary disk instead of a full-size in-memory copy; retained output still scales with extracted content. ## 0.27.11 diff --git a/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py b/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py index d983e3539f..7df5bd0f0e 100644 --- a/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py +++ b/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py @@ -162,3 +162,39 @@ def test_single_page_render_matches_native_batch_pixels_and_metadata(fixture): assert candidate.tobytes() == baseline.tobytes() baseline.close() candidate.close() + + +class ReadSeekStream(io.RawIOBase): + """A readable, seekable stream that is not a `BytesIO`, spool or buffered file.""" + + def __init__(self, payload: bytes): + self._buffer = io.BytesIO(payload) + + def readable(self): + return True + + def seekable(self): + return True + + def readinto(self, buffer): + return self._buffer.readinto(buffer) + + def seek(self, offset, whence=io.SEEK_SET): + return self._buffer.seek(offset, whence) + + def tell(self): + return self._buffer.tell() + + +def test_ocr_only_accepts_any_readable_seekable_stream(): + from test_unstructured.unit_utils import example_doc_path + from unstructured.partition import pdf + + path = example_doc_path("pdf/layout-parser-paper-fast.pdf") + expected = pdf._partition_pdf_or_image_with_ocr(filename=path) + source = ReadSeekStream(Path(path).read_bytes()) + source.seek(5) + actual = pdf._partition_pdf_or_image_with_ocr(file=source) + assert [element.text for element in actual] == [element.text for element in expected] + assert not source.closed + assert source.tell() == 0 diff --git a/unstructured/partition/pdf_image/pdf_image_utils.py b/unstructured/partition/pdf_image/pdf_image_utils.py index 800d170d2b..5973c01bd5 100644 --- a/unstructured/partition/pdf_image/pdf_image_utils.py +++ b/unstructured/partition/pdf_image/pdf_image_utils.py @@ -426,7 +426,8 @@ def convert_pdf_to_images( def _pdf_source_path(filename: str, file: Optional[bytes | IO[bytes]]) -> Iterator[str]: """Yield a filesystem path for `filename` or a temporary on-disk copy of `file`. - A `BytesIO` cursor is restored after the copy and a `SpooledTemporaryFile` is left rewound. + Any readable, seekable `file` is accepted. A `BytesIO` cursor is restored after the copy and + other streams are left rewound; buffered files on disk are re-read by name. """ if file is None: yield filename @@ -436,16 +437,16 @@ def _pdf_source_path(filename: str, file: Optional[bytes | IO[bytes]]) -> Iterat with open(path, "wb") as destination: if isinstance(file, bytes): destination.write(file) - elif isinstance(file, (BytesIO, tempfile.SpooledTemporaryFile)): - position = 0 if isinstance(file, tempfile.SpooledTemporaryFile) else file.tell() + elif isinstance(file, (TextIOWrapper, BufferedReader)): + with open(file.name, "rb") as source: + shutil.copyfileobj(source, destination, length=1024 * 1024) + elif callable(getattr(file, "read", None)) and callable(getattr(file, "seek", None)): + position = file.tell() if isinstance(file, BytesIO) else 0 try: file.seek(0) shutil.copyfileobj(file, destination, length=1024 * 1024) finally: file.seek(position) - elif isinstance(file, (TextIOWrapper, BufferedReader)): - with open(file.name, "rb") as source: - shutil.copyfileobj(source, destination, length=1024 * 1024) else: raise ValueError("Invalid file-like object type") yield path From e5a7f4c99016d47c4d76f8715b97bf4c5ce2068e Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:02:43 -0400 Subject: [PATCH 6/7] docs(changelog): scope the OCR upload memory claim --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fbcce1a40..a140b998ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Fixes -- **Bound PDF OCR upload and render memory.** Copy uploaded OCR inputs to temporary files in 1 MiB blocks, render OCR-only PDFs one page at a time, and close owned page images after OCR. Uploads may be any readable, seekable stream. Preserve native rendered pixels, image metadata, output order and caller-owned upload streams. Large uploaded PDFs now use temporary disk instead of a full-size in-memory copy; retained output still scales with extracted content. +- **Bound PDF OCR upload and render memory.** Copy uploaded OCR inputs to temporary files in 1 MiB blocks, render OCR-only PDFs one page at a time, and close owned page images after OCR. Uploads may be any readable, seekable stream. Preserve native rendered pixels, image metadata, output order and caller-owned upload streams. OCR rendering reads uploads from temporary disk instead of holding a full-size in-memory copy; retained output still scales with extracted content. ## 0.27.11 From 61adb350557045eecaa162817170cfc215b2e779 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:25:39 -0400 Subject: [PATCH 7/7] fix(pdf): copy nameless buffered uploads through read and seek --- .../partition/pdf_image/test_pdf_streaming_memory.py | 10 ++++++++++ unstructured/partition/pdf_image/pdf_image_utils.py | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py b/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py index 7df5bd0f0e..b2c7419fc5 100644 --- a/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py +++ b/test_unstructured/partition/pdf_image/test_pdf_streaming_memory.py @@ -198,3 +198,13 @@ def test_ocr_only_accepts_any_readable_seekable_stream(): assert [element.text for element in actual] == [element.text for element in expected] assert not source.closed assert source.tell() == 0 + + +def test_nameless_buffered_stream_is_copied_through_read_and_seek(): + payload = b"document" * 300000 + source = io.BufferedReader(io.BytesIO(payload)) + source.seek(3) + with pdf_image_utils._pdf_source_path("", source) as copied_path: + assert Path(copied_path).read_bytes() == payload + assert not source.closed + assert source.tell() == 0 diff --git a/unstructured/partition/pdf_image/pdf_image_utils.py b/unstructured/partition/pdf_image/pdf_image_utils.py index 5973c01bd5..a64c092176 100644 --- a/unstructured/partition/pdf_image/pdf_image_utils.py +++ b/unstructured/partition/pdf_image/pdf_image_utils.py @@ -427,7 +427,7 @@ def _pdf_source_path(filename: str, file: Optional[bytes | IO[bytes]]) -> Iterat """Yield a filesystem path for `filename` or a temporary on-disk copy of `file`. Any readable, seekable `file` is accepted. A `BytesIO` cursor is restored after the copy and - other streams are left rewound; buffered files on disk are re-read by name. + other streams are left rewound; buffered files with a path name are re-read from that path. """ if file is None: yield filename @@ -437,7 +437,9 @@ def _pdf_source_path(filename: str, file: Optional[bytes | IO[bytes]]) -> Iterat with open(path, "wb") as destination: if isinstance(file, bytes): destination.write(file) - elif isinstance(file, (TextIOWrapper, BufferedReader)): + elif isinstance(file, (TextIOWrapper, BufferedReader)) and isinstance( + getattr(file, "name", None), str + ): with open(file.name, "rb") as source: shutil.copyfileobj(source, destination, length=1024 * 1024) elif callable(getattr(file, "read", None)) and callable(getattr(file, "seek", None)):