Add implementation of streaming abstractions

HISTORY.md

@@ -2,6 +2,8 @@
 
 ## UNRELEASED
 
+- Added streaming I/O via `FileCacheMode.streaming` for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS. In this mode `CloudPath.open()` returns the standard `io` file objects (`BufferedReader`/`BufferedWriter`/`TextIOWrapper`) backed by ranged reads and multipart uploads instead of the local cache, so only the bytes you read are downloaded and written data is uploaded as you write it. Read streams are seekable; append and update modes fall back to the cache and remove the cache file on close. `buffering` has its builtin `open()` meaning and sets the ranged-request size (default 5 MiB). A new `streaming_max_concurrency` client option (default 4) bounds background read-ahead and part uploads per stream; the buffer size, concurrency, and multipart part size can also be set with the `CLOUDPATHLIB_STREAMING_BUFFER_SIZE`, `CLOUDPATHLIB_STREAMING_MAX_CONCURRENCY`, and `CLOUDPATHLIB_[<PROVIDER>_]STREAMING_PART_SIZE` environment variables. Opening a file for append or update in streaming mode warns that it falls back to the cache. `copy`/`rename`/`replace` stream between clients, `fspath` raises `CloudPathNotImplementedError`, and failures raise `cloudpathlib.exceptions` types including the new `CloudPathStreamingError`. See the [Streaming I/O](https://cloudpathlib.drivendata.org/stable/streaming_io/) docs. (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535))
+
 ## v0.26.0 (2026-10-01)
 
 - Added support for Python 3.15: `CloudPath.mkdir(parent_mode=...)` works across providers, and cloud paths can be pickled repeatedly without a `ParseResult` error. (Issue [#593](https://github.com/drivendataorg/cloudpathlib/issues/593), PR [#594](https://github.com/drivendataorg/cloudpathlib/pull/594))

cloudpathlib/azure/azblobclient.py

@@ -4,12 +4,26 @@
 import os
 from http import HTTPStatus
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union, cast
-
-from ..client import Client, register_client_class
+from typing import (
+    Any,
+    BinaryIO,
+    Callable,
+    Dict,
+    Iterable,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
+from uuid import uuid4
+
+from ..client import Client, _UploadPart, register_client_class
+from ..env import MiB
+from ..cloud_io import _CloudMultipartStorageRaw
 from ..cloudpath import implementation_registry
 from ..enums import FileCacheMode
-from ..exceptions import MissingCredentialsError
+from ..exceptions import CloudPathFileNotFoundError, MissingCredentialsError
 from .azblobpath import AzureBlobPath
 
 try:
@@ -46,6 +60,13 @@ class AzureBlobClient(Client):
     authentication options.
     """
 
+    _streaming_raw_class = _CloudMultipartStorageRaw
+    # Azure block blobs: 4 MiB default block (the SDK's default), 4000 MiB largest block,
+    # 50,000 committed blocks per blob
+    _multipart_min_part_size = 4 * MiB
+    _multipart_max_part_size = 4_000 * MiB
+    _multipart_max_parts = 50_000
+
     def __init__(
         self,
         account_url: Optional[str] = None,
@@ -56,6 +77,8 @@ def __init__(
         file_cache_mode: Optional[Union[str, FileCacheMode]] = None,
         local_cache_dir: Optional[Union[str, os.PathLike]] = None,
         content_type_method: Optional[Callable] = mimetypes.guess_type,
+        *,
+        streaming_max_concurrency: Optional[int] = None,
     ):
         """Class constructor. Sets up a [`BlobServiceClient`](
         https://docs.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobserviceclient?view=azure-python).
@@ -103,11 +126,17 @@ def __init__(
                 the `CLOUDPATHLIB_LOCAL_CACHE_DIR` environment variable.
             content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when
                 writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding).
+            streaming_max_concurrency (Optional[int]): Maximum concurrent requests per open
+                streaming file (background part uploads while writing, read-ahead of the next
+                byte ranges during sequential reads) when using `FileCacheMode.streaming`.
+                Defaults to the `CLOUDPATHLIB_STREAMING_MAX_CONCURRENCY` environment variable
+                or 4; 1 makes each stream fully sequential.
         """
         super().__init__(
             local_cache_dir=local_cache_dir,
             content_type_method=content_type_method,
             file_cache_mode=file_cache_mode,
+            streaming_max_concurrency=streaming_max_concurrency,
         )
 
         if connection_string is None:
@@ -456,22 +485,21 @@ def _upload_file(
             container=cloud_path.container, blob=cloud_path.blob
         )
 
-        extra_args = {}
-        if self.content_type_method is not None:
-            content_type, content_encoding = self.content_type_method(str(local_path))
-
-            if content_type is not None:
-                extra_args["content_type"] = content_type
-            if content_encoding is not None:
-                extra_args["content_encoding"] = content_encoding
-
-        content_settings = ContentSettings(**extra_args)
-
         with Path(local_path).open("rb") as data:
-            blob.upload_blob(data, overwrite=True, content_settings=content_settings)  # type: ignore
+            blob.upload_blob(
+                data, overwrite=True, content_settings=self._content_settings(cloud_path)
+            )  # type: ignore
 
         return cloud_path
 
+    def _content_settings(self, cloud_path: AzureBlobPath) -> Optional["ContentSettings"]:
+        """The guessed content type/encoding (see `Client._guess_content_type`) in the form the
+        Azure SDK takes."""
+        content_type, content_encoding = self._guess_content_type(cloud_path)
+        if content_type is None and content_encoding is None:
+            return None
+        return ContentSettings(content_type=content_type, content_encoding=content_encoding)
+
     def _get_public_url(self, cloud_path: AzureBlobPath) -> str:
         blob_client = self.service_client.get_blob_client(
             container=cloud_path.container, blob=cloud_path.blob
@@ -492,6 +520,79 @@ def _generate_presigned_url(
         url = f"{self._get_public_url(cloud_path)}?{sas_token}"
         return url
 
+    def _range_download(
+        self, cloud_path: AzureBlobPath, start: int, end: Optional[int] = None
+    ) -> bytes:
+        """Download a byte range from Azure Blob Storage."""
+        blob_client = self.service_client.get_blob_client(
+            container=cloud_path.container, blob=cloud_path.blob
+        )
+        try:
+            length = None if end is None else end - start + 1
+            downloader = blob_client.download_blob(offset=start, length=length)
+            return downloader.readall()
+        except ResourceNotFoundError as e:
+            raise CloudPathFileNotFoundError(f"Azure blob not found: {cloud_path}") from e
+        except HttpResponseError as e:
+            # a range starting past the end of the blob is how a reader that does not know
+            # the size learns it is at EOF: file objects report that as an empty read
+            if (e.error and e.error.code == "InvalidRange") or e.status_code == 416:
+                return b""
+            raise
+
+    def _initiate_multipart_upload(self, cloud_path: AzureBlobPath) -> str:
+        """Return a unique session ID that namespaces this upload's block IDs.
+
+        Azure has no "begin upload" call; uncommitted blocks live in one per-blob namespace
+        keyed by block ID. A random session ID (rather than IDs derived from the content or
+        part number alone) keeps two writers streaming to the same blob at the same time
+        from overwriting each other's staged blocks and committing interleaved data.
+        """
+        return uuid4().hex
+
+    def _upload_part(
+        self, cloud_path: AzureBlobPath, upload_id: str, part_number: int, data: bytes
+    ) -> _UploadPart:
+        """Upload a block in an Azure block blob upload."""
+        blob_client = self.service_client.get_blob_client(
+            container=cloud_path.container, blob=cloud_path.blob
+        )
+        # Azure requires all block IDs of a blob to have the same length; uuid4().hex (32)
+        # plus a fixed-width part number keeps them uniform. The SDK base64-encodes the ID.
+        block_id = f"{upload_id}-{part_number:06d}"
+        blob_client.stage_block(block_id=block_id, data=data, length=len(data))
+        return {"block_id": block_id}
+
+    def _complete_multipart_upload(
+        self, cloud_path: AzureBlobPath, upload_id: str, parts: Sequence[_UploadPart]
+    ) -> None:
+        """Commit an Azure block blob upload, threading content-type."""
+        blob_client = self.service_client.get_blob_client(
+            container=cloud_path.container, blob=cloud_path.blob
+        )
+        block_ids = [part["block_id"] for part in parts]
+        blob_client.commit_block_list(
+            block_ids, content_settings=self._content_settings(cloud_path)
+        )
+
+    def _abort_multipart_upload(self, cloud_path: AzureBlobPath, upload_id: str) -> None:
+        """Nothing to do: Azure has no call to discard uncommitted blocks.
+
+        The service garbage-collects blocks that are not committed within a week (see the
+        Put Block documentation, https://learn.microsoft.com/rest/api/storageservices/put-block).
+        The only way to drop them sooner is to commit a block list without them, which would
+        create or rewrite the blob, so it is not done here.
+        """
+
+    def _put_object(self, cloud_path: AzureBlobPath, data: BinaryIO) -> None:
+        """Upload a whole blob in one request, threading content-type."""
+        blob_client = self.service_client.get_blob_client(
+            container=cloud_path.container, blob=cloud_path.blob
+        )
+        blob_client.upload_blob(
+            data, overwrite=True, content_settings=self._content_settings(cloud_path)
+        )
+
 
 def _hns_rmtree(data_lake_client, container, directory):
     """Stateless implementation so can be used in test suite cleanup as well.

cloudpathlib/client.py

@@ -4,13 +4,34 @@
 from pathlib import Path
 import shutil
 from tempfile import TemporaryDirectory
-from typing import ClassVar, Generic, Callable, Iterable, Optional, Tuple, TypeVar, Union
-
+from typing import (
+    Any,
+    BinaryIO,
+    Callable,
+    ClassVar,
+    Dict,
+    Generic,
+    Iterable,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
+
+from . import env
+from .cloud_io import _CloudStorageRaw
 from .cloudpath import CloudImplementation, CloudPath, implementation_registry
 from .enums import FileCacheMode
-from .exceptions import InvalidConfigurationException
+from .exceptions import (
+    CloudPathFileNotFoundError,
+    InvalidConfigurationException,
+    NoStatError,
+)
 
 BoundedCloudPath = TypeVar("BoundedCloudPath", bound=CloudPath)
+_UploadPart = Dict[str, Any]
 
 
 def register_client_class(key: str) -> Callable:
@@ -29,16 +50,45 @@ class Client(abc.ABC, Generic[BoundedCloudPath]):
     _cloud_meta: CloudImplementation
     _default_client: ClassVar[Optional["Client[BoundedCloudPath]"]] = None
 
+    # Streaming I/O (`FileCacheMode.streaming`): the raw stream class `CloudPath.open` wraps,
+    # or None when the provider has no streaming support. Multipart providers use
+    # `_CloudMultipartStorageRaw` and the provider's hard limits below (S3's by default).
+    # The part size actually used starts at the minimum (overridable per client with
+    # `CLOUDPATHLIB_<PROVIDER>_STREAMING_PART_SIZE`, see `env.py`) and grows for very large
+    # streams so the part-count limit is never reached.
+    _streaming_raw_class: ClassVar[Optional[Type[_CloudStorageRaw]]] = None
+    _multipart_min_part_size: ClassVar[int] = 5 * env.MiB  # smallest non-final part (5 MiB)
+    _multipart_max_part_size: ClassVar[int] = 5 * 1024 * env.MiB  # largest part (5 GiB)
+    _multipart_max_parts: ClassVar[int] = 10_000
+
     def __init__(
         self,
         file_cache_mode: Optional[Union[str, FileCacheMode]] = None,
         local_cache_dir: Optional[Union[str, os.PathLike]] = None,
         content_type_method: Optional[Callable] = mimetypes.guess_type,
-    ):
+        *,
+        streaming_max_concurrency: Optional[int] = None,
+    ) -> None:
         self.file_cache_mode = None
         self._cache_tmp_dir = None
         self._cloud_meta.validate_completeness()
 
+        # concurrent requests per open streaming file (part uploads / read-ahead);
+        # 1 means fully sequential I/O
+        if streaming_max_concurrency is None:
+            streaming_max_concurrency = env.streaming_max_concurrency()
+        if (
+            not isinstance(streaming_max_concurrency, int)
+            or isinstance(streaming_max_concurrency, bool)
+            or streaming_max_concurrency < 1
+        ):
+            raise ValueError("streaming_max_concurrency must be an integer of at least 1")
+        self.streaming_max_concurrency = streaming_max_concurrency
+        # multipart part size for streaming writes: the provider minimum unless overridden
+        self._multipart_part_size = env.streaming_part_size(
+            getattr(self._cloud_meta, "name", None), self._multipart_min_part_size
+        )
+
         # convert strings passed to enum
         if isinstance(file_cache_mode, str):
             file_cache_mode = FileCacheMode(file_cache_mode)
@@ -88,6 +138,9 @@ def __del__(self) -> None:
             FileCacheMode.tmp_dir,
             FileCacheMode.close_file,
             FileCacheMode.cloudpath_object,
+            # streaming avoids the cache except for append/update fallbacks, which
+            # should not outlive the client
+            FileCacheMode.streaming,
         ]:
             self.clear_cache()
 
@@ -175,6 +228,15 @@ def _upload_file(
     ) -> BoundedCloudPath:
         pass
 
+    def _guess_content_type(
+        self, cloud_path: BoundedCloudPath
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """`(content_type, content_encoding)` for an upload, from `content_type_method` applied
+        to the object name; `(None, None)` when guessing is disabled."""
+        if self.content_type_method is None:
+            return None, None
+        return self.content_type_method(str(cloud_path))
+
     @abc.abstractmethod
     def _get_public_url(self, cloud_path: BoundedCloudPath) -> str:
         pass
@@ -184,3 +246,58 @@ def _generate_presigned_url(
         self, cloud_path: BoundedCloudPath, expire_seconds: int = 60 * 60
     ) -> str:
         pass
+
+    def _range_download(
+        self, cloud_path: BoundedCloudPath, start: int, end: Optional[int] = None
+    ) -> bytes:
+        """Download the inclusive byte range `start`-`end`, or from `start` to the end of the
+        object when `end` is None."""
+        raise NotImplementedError(
+            f"{type(self).__name__} does not support streaming I/O (_range_download). "
+            "Implement this method or use a non-streaming file_cache_mode."
+        )
+
+    def _get_content_length(self, cloud_path: BoundedCloudPath) -> Optional[int]:
+        """Object size without downloading it, or None when the provider cannot say.
+
+        `stat()` is one metadata request on every provider (HEAD-equivalent through
+        `_get_metadata`), so this is as cheap as a dedicated size call would be.
+        """
+        try:
+            return cloud_path.stat().st_size
+        except NoStatError as e:
+            raise CloudPathFileNotFoundError(f"Object not found: {cloud_path}") from e
+
+    def _initiate_multipart_upload(self, cloud_path: BoundedCloudPath) -> str:
+        """Start a multipart upload."""
+        raise NotImplementedError(
+            f"{type(self).__name__} does not support streaming I/O (_initiate_multipart_upload)."
+        )
+
+    def _upload_part(
+        self, cloud_path: BoundedCloudPath, upload_id: str, part_number: int, data: bytes
+    ) -> _UploadPart:
+        """Upload one part."""
+        raise NotImplementedError(
+            f"{type(self).__name__} does not support streaming I/O (_upload_part)."
+        )
+
+    def _complete_multipart_upload(
+        self, cloud_path: BoundedCloudPath, upload_id: str, parts: Sequence[_UploadPart]
+    ) -> None:
+        """Complete a multipart upload."""
+        raise NotImplementedError(
+            f"{type(self).__name__} does not support streaming I/O (_complete_multipart_upload)."
+        )
+
+    def _abort_multipart_upload(self, cloud_path: BoundedCloudPath, upload_id: str) -> None:
+        """Abort a multipart upload."""
+        raise NotImplementedError(
+            f"{type(self).__name__} does not support streaming I/O (_abort_multipart_upload)."
+        )
+
+    def _put_object(self, cloud_path: BoundedCloudPath, data: BinaryIO) -> None:
+        """Upload a whole object in one request from a readable binary stream."""
+        raise NotImplementedError(
+            f"{type(self).__name__} does not support streaming I/O (_put_object)."
+        )