From f2b79bdec81ae203231b9ca7468f124289f7b332 Mon Sep 17 00:00:00 2001 From: terminalchai <213856599+terminalchai@users.noreply.github.com> Date: Fri, 16 Oct 2026 03:45:47 +0530 Subject: [PATCH 1/2] feat: support error handling in CloudPath.rmtree --- HISTORY.md | 1 + cloudpathlib/cloudpath.py | 44 ++++++++++-- tests/test_rmtree.py | 141 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 tests/test_rmtree.py diff --git a/HISTORY.md b/HISTORY.md index d65f243d..4cee9a9a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -3,6 +3,7 @@ ## UNRELEASED - Added streaming I/O via `FileCacheMode.streaming` for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS. In this mode `CloudPath.open()` returns the standard `io` file objects (`BufferedReader`/`BufferedWriter`/`TextIOWrapper`) backed by ranged reads and multipart uploads instead of the local cache, so only the bytes you read are downloaded and written data is uploaded as you write it. Read streams are seekable; append and update modes fall back to the cache and remove the cache file on close. `buffering` has its builtin `open()` meaning and sets the ranged-request size (default 5 MiB). A new `streaming_max_concurrency` client option (default 4) bounds background read-ahead and part uploads per stream; the buffer size, concurrency, and multipart part size can also be set with the `CLOUDPATHLIB_STREAMING_BUFFER_SIZE`, `CLOUDPATHLIB_STREAMING_MAX_CONCURRENCY`, and `CLOUDPATHLIB_[_]STREAMING_PART_SIZE` environment variables. Opening a file for append or update in streaming mode warns that it falls back to the cache. `copy`/`rename`/`replace` stream between clients, `fspath` raises `CloudPathNotImplementedError`, and failures raise `cloudpathlib.exceptions` types including the new `CloudPathStreamingError`. See the [Streaming I/O](https://cloudpathlib.drivendata.org/stable/streaming_io/) docs. (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) +- Added `ignore_errors`, `onerror`, and `onexc` options to `CloudPath.rmtree()` for suppressing deletion errors or handling them with a callback. Calls without these options still raise errors. (Issue [#270](https://github.com/drivendataorg/cloudpathlib/issues/270)) - Changed append mode (`a`, `ab`, `a+`, and `ab+`) to create missing cloud files, matching Python file behavior. Appending to existing files still preserves their content. (Issue [#585](https://github.com/drivendataorg/cloudpathlib/issues/585)) - Changed test dependencies to permit newer `pytest-rerunfailures` releases while excluding version 16.0, which causes distributed-test serialization failures. (Issue [#528](https://github.com/drivendataorg/cloudpathlib/issues/528), PR [#606](https://github.com/drivendataorg/cloudpathlib/pull/606)) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index eb1fc50e..fb3c0b68 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -1315,13 +1315,43 @@ def download_to(self, destination: Union[str, os.PathLike]) -> Path: return destination - def rmtree(self) -> None: - """Delete an entire directory tree.""" - if self.is_file(): - raise CloudPathNotADirectoryError( - f"Path {self} is a file; call unlink instead of rmtree." - ) - self.client._remove(self) + def rmtree( + self, + ignore_errors: bool = False, + onerror: Optional[Callable] = None, + *, + onexc: Optional[Callable] = None, + ) -> None: + """Delete an entire directory tree. + + By default, exceptions from checking the path or removing cloud objects are raised. + Set ``ignore_errors=True`` to suppress these exceptions. This takes precedence over + either callback. Control-flow exceptions such as ``KeyboardInterrupt`` are not suppressed. + + ``onexc(function, path, exception)`` handles a failure with the exception object; + ``onerror(function, path, exc_info)`` receives the tuple returned by ``sys.exc_info()``. + If both callbacks are supplied, ``onexc`` takes precedence, matching ``shutil.rmtree``. + Exceptions raised by either callback propagate to the caller. + + Callbacks receive the unbound ``rmtree`` method and this cloud path, so + ``function(path)`` retries the entire operation. Callbacks run once per failed operation, + rather than once per object in the tree; a backend may already have removed some objects. + """ + try: + if self.is_file(): + raise CloudPathNotADirectoryError( + f"Path {self} is a file; call unlink instead of rmtree." + ) + self.client._remove(self) + except Exception as error: + if ignore_errors: + return + if onexc is not None: + onexc(type(self).rmtree, self, error) + elif onerror is not None: + onerror(type(self).rmtree, self, sys.exc_info()) + else: + raise def upload_from( self, diff --git a/tests/test_rmtree.py b/tests/test_rmtree.py new file mode 100644 index 00000000..7829437e --- /dev/null +++ b/tests/test_rmtree.py @@ -0,0 +1,141 @@ +from types import TracebackType + +import pytest + +from cloudpathlib.exceptions import CloudPathNotADirectoryError + + +@pytest.fixture(params=["is_file", "remove"]) +def failed_rmtree(request, rig, monkeypatch): + path = rig.create_cloud_path("dir_1/") + error = RuntimeError("cloud operation failed") + + def fail(*args, **kwargs): + raise error + + if request.param == "is_file": + monkeypatch.setattr(path, "is_file", fail) + else: + monkeypatch.setattr(path.client, "_remove", fail) + return path, error + + +@pytest.mark.parametrize("options", [{}, {"ignore_errors": False}]) +def test_rmtree_propagates_errors_by_default(failed_rmtree, options): + path, error = failed_rmtree + with pytest.raises(RuntimeError) as exc: + path.rmtree(**options) + assert exc.value is error + + +def test_rmtree_ignore_errors(failed_rmtree): + path, _ = failed_rmtree + assert path.rmtree(ignore_errors=True) is None + assert path.exists() + + +@pytest.mark.parametrize("handler", ["onerror", "onexc", "both", "ignore_errors"]) +def test_rmtree_error_callbacks(failed_rmtree, handler): + path, error = failed_rmtree + calls = [] + + def onerror(function, failed_path, exc_info): + calls.append(("onerror", function, failed_path, exc_info)) + + def onexc(function, failed_path, exception): + calls.append(("onexc", function, failed_path, exception)) + + options = {} + if handler in ("onerror", "both", "ignore_errors"): + options["onerror"] = onerror + if handler in ("onexc", "both", "ignore_errors"): + options["onexc"] = onexc + if handler == "ignore_errors": + options["ignore_errors"] = True + + assert path.rmtree(**options) is None + if handler == "ignore_errors": + assert calls == [] + else: + assert len(calls) == 1 + name, function, failed_path, detail = calls[0] + assert name == ("onerror" if handler == "onerror" else "onexc") + assert function is type(path).rmtree + assert failed_path is path + if name == "onerror": + assert detail[:2] == (RuntimeError, error) + assert isinstance(detail[2], TracebackType) + else: + assert detail is error + assert path.exists() + + +@pytest.mark.parametrize("handler", ["onerror", "onexc"]) +def test_rmtree_callback_errors_propagate(failed_rmtree, handler): + path, _ = failed_rmtree + callback_error = ValueError("callback failed") + + def callback(*args): + raise callback_error + + with pytest.raises(ValueError) as exc: + path.rmtree(**{handler: callback}) + assert exc.value is callback_error + + +@pytest.mark.parametrize("error_type", [KeyboardInterrupt, SystemExit]) +def test_rmtree_ignore_errors_does_not_suppress_control_flow(rig, monkeypatch, error_type): + path = rig.create_cloud_path("dir_1/") + error = error_type() + + def fail(*args): + raise error + + monkeypatch.setattr(path.client, "_remove", fail) + with pytest.raises(error_type) as exc: + path.rmtree(ignore_errors=True) + assert exc.value is error + + +@pytest.mark.parametrize("option", ["ignore_errors", "onerror", "onexc"]) +def test_rmtree_success_with_error_handling(rig, option): + path = rig.create_cloud_path("dir_1/") + + def unexpected_callback(*args): + pytest.fail("The callback should not be called after a successful removal") + + options = {option: True if option == "ignore_errors" else unexpected_callback} + assert path.rmtree(**options) is None + assert not path.exists() + + +@pytest.mark.parametrize("ignore_errors", [False, True]) +def test_rmtree_file_error(rig, ignore_errors): + path = rig.create_cloud_path("dir_0/file0_0.txt") + if ignore_errors: + assert path.rmtree(ignore_errors=True) is None + else: + with pytest.raises(CloudPathNotADirectoryError): + path.rmtree(ignore_errors=False) + assert path.exists() + + +@pytest.mark.parametrize("handler", ["onerror", "onexc"]) +def test_rmtree_callback_can_retry(rig, monkeypatch, handler): + path = rig.create_cloud_path("dir_1/") + remove = path.client._remove + attempts = [] + + def fail_once(failed_path): + attempts.append(failed_path) + if len(attempts) == 1: + raise PermissionError("temporary removal failure") + return remove(failed_path) + + def retry(function, failed_path, error): + function(failed_path) + + monkeypatch.setattr(path.client, "_remove", fail_once) + path.rmtree(**{handler: retry}) + assert attempts == [path, path] + assert not path.exists() From 2c31a1f07e85f10ef3ee402fff2301cca0f3b7a2 Mon Sep 17 00:00:00 2001 From: terminalchai <213856599+terminalchai@users.noreply.github.com> Date: Fri, 16 Oct 2026 03:47:02 +0530 Subject: [PATCH 2/2] docs: link rmtree error handling to PR 604 --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 4cee9a9a..d5de5419 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -3,7 +3,7 @@ ## UNRELEASED - Added streaming I/O via `FileCacheMode.streaming` for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS. In this mode `CloudPath.open()` returns the standard `io` file objects (`BufferedReader`/`BufferedWriter`/`TextIOWrapper`) backed by ranged reads and multipart uploads instead of the local cache, so only the bytes you read are downloaded and written data is uploaded as you write it. Read streams are seekable; append and update modes fall back to the cache and remove the cache file on close. `buffering` has its builtin `open()` meaning and sets the ranged-request size (default 5 MiB). A new `streaming_max_concurrency` client option (default 4) bounds background read-ahead and part uploads per stream; the buffer size, concurrency, and multipart part size can also be set with the `CLOUDPATHLIB_STREAMING_BUFFER_SIZE`, `CLOUDPATHLIB_STREAMING_MAX_CONCURRENCY`, and `CLOUDPATHLIB_[_]STREAMING_PART_SIZE` environment variables. Opening a file for append or update in streaming mode warns that it falls back to the cache. `copy`/`rename`/`replace` stream between clients, `fspath` raises `CloudPathNotImplementedError`, and failures raise `cloudpathlib.exceptions` types including the new `CloudPathStreamingError`. See the [Streaming I/O](https://cloudpathlib.drivendata.org/stable/streaming_io/) docs. (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) -- Added `ignore_errors`, `onerror`, and `onexc` options to `CloudPath.rmtree()` for suppressing deletion errors or handling them with a callback. Calls without these options still raise errors. (Issue [#270](https://github.com/drivendataorg/cloudpathlib/issues/270)) +- Added `ignore_errors`, `onerror`, and `onexc` options to `CloudPath.rmtree()` for suppressing deletion errors or handling them with a callback. Calls without these options still raise errors. (Issue [#270](https://github.com/drivendataorg/cloudpathlib/issues/270), PR [#604](https://github.com/drivendataorg/cloudpathlib/pull/604)) - Changed append mode (`a`, `ab`, `a+`, and `ab+`) to create missing cloud files, matching Python file behavior. Appending to existing files still preserves their content. (Issue [#585](https://github.com/drivendataorg/cloudpathlib/issues/585)) - Changed test dependencies to permit newer `pytest-rerunfailures` releases while excluding version 16.0, which causes distributed-test serialization failures. (Issue [#528](https://github.com/drivendataorg/cloudpathlib/issues/528), PR [#606](https://github.com/drivendataorg/cloudpathlib/pull/606))