diff --git a/HISTORY.md b/HISTORY.md index 1ae9db02..3536de46 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -8,6 +8,8 @@ - Fixed mypy missing-import errors for `CloudPath` and `AnyPath` when the optional Pydantic integration is not installed. (Issue [#385](https://github.com/drivendataorg/cloudpathlib/issues/385), PR [#603](https://github.com/drivendataorg/cloudpathlib/pull/603)) +- Fixed stale local caches after cloud file and directory deletions and moves, including `move`, `move_into`, and ADLS Gen2 directory renames. Clean cached files are also discarded after external cloud deletions, so exclusive creation and append start fresh. (Issue [#388](https://github.com/drivendataorg/cloudpathlib/issues/388), PR [#600](https://github.com/drivendataorg/cloudpathlib/pull/600)) + ## v0.26.0 (2026-10-01) - Added support for Python 3.15: `CloudPath.mkdir(parent_mode=...)` works across providers, and cloud paths can be pickled repeatedly without a `ParseResult` error. (Issue [#593](https://github.com/drivendataorg/cloudpathlib/issues/593), PR [#594](https://github.com/drivendataorg/cloudpathlib/pull/594)) diff --git a/cloudpathlib/azure/azblobpath.py b/cloudpathlib/azure/azblobpath.py index 1c5c4bb0..8a7163f2 100644 --- a/cloudpathlib/azure/azblobpath.py +++ b/cloudpathlib/azure/azblobpath.py @@ -87,7 +87,12 @@ def replace(self, target: "AzureBlobPath") -> "AzureBlobPath": # we can rename directories on ADLS Gen2 except CloudPathIsADirectoryError: if self.client._check_hns(self): - return self.client._move_file(self, target) + if target == self: + return self + result = self.client._move_file(self, target) + self.clear_cache() + target.clear_cache() + return result else: raise diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index eb1fc50e..a6829861 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -1010,6 +1010,8 @@ def replace(self, target: Self) -> Self: target.unlink() self.client._move_file(self, target) + self.clear_cache() + target.clear_cache() return target def rename(self, target: Self) -> Self: @@ -1030,6 +1032,7 @@ def rmdir(self) -> None: except StopIteration: pass self.client._remove(self) + self.clear_cache() def samefile(self, other_path: Union[str, os.PathLike]) -> bool: # all cloud paths are absolute and the paths are used for hash @@ -1042,6 +1045,7 @@ def unlink(self, missing_ok: bool = True) -> None: f"Path {self} is a directory; call rmdir instead of unlink." ) self.client._remove(self, missing_ok) + self.clear_cache() def write_bytes(self, data: bytes) -> int: """Open the file in bytes mode, write to it, and close the file. @@ -1322,6 +1326,7 @@ def rmtree(self) -> None: f"Path {self} is a file; call unlink instead of rmtree." ) self.client._remove(self) + self.clear_cache() def upload_from( self, @@ -1375,9 +1380,16 @@ def _copy( if destination.exists() and destination.is_dir(): destination = destination / self.name + if remove_src: + if destination == self: + return self self._raise_if_copy_target_newer(destination, force_overwrite_to_cloud) - return cast(Self, self.client._move_file(self, destination, remove_src=remove_src)) + result = self.client._move_file(self, destination, remove_src=remove_src) + if remove_src: + self.clear_cache() + destination.clear_cache() + return cast(Self, result) else: if not destination.exists() or destination.is_file(): @@ -1706,8 +1718,10 @@ def _refresh_cache(self, force_overwrite_from_cloud: Optional[bool] = None) -> N try: stats = self.stat() except NoStatError: - # nothing to cache if the file does not exist; happens when creating - # new files that will be uploaded + # Discard stale files after external deletions, retaining pending writes + # and cached directory contents that cannot be described by a file stat. + if not self._dirty and self._local.is_file(): + self.clear_cache() return # if not exist or cloud newer diff --git a/tests/test_caching.py b/tests/test_caching.py index 2bc93658..3b48d22c 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -13,6 +13,7 @@ from cloudpathlib.enums import FileCacheMode from cloudpathlib.exceptions import ( InvalidConfigurationException, + NoStatError, OverwriteNewerCloudError, OverwriteNewerLocalError, ) @@ -580,3 +581,257 @@ def test_streaming_append_fallback_cache_cleaned_up(rig: CloudProviderTestRig): assert not cp._local.exists() assert cp.read_text() == original + "appended" + + +@pytest.mark.parametrize("mode", ["x", "a"]) +def test_unlink_clears_cache_before_recreating(rig: CloudProviderTestRig, mode): + cp = rig.create_cloud_path("unlink-cache.txt") + cp.write_text("old content") + assert cp._local.exists() + + cp.unlink() + + assert not cp.exists() + assert not cp._local.exists() + with cp.open(mode) as handle: + handle.write("new content") + assert cp.read_text() == "new content" + + +@pytest.mark.parametrize("method", ["rename", "replace"]) +@pytest.mark.parametrize("target_exists", [False, True]) +def test_move_clears_source_and_target_caches(rig: CloudProviderTestRig, method, target_exists): + source = rig.create_cloud_path("move-source.txt") + target = rig.create_cloud_path("move-target.txt") + source.write_text("source content") + target.write_text("old target content") + if not target_exists: + # Model a remote deletion made outside cloudpathlib, leaving a stale target cache. + target.client._remove(target) + assert source._local.exists() + assert target._local.exists() + + result = getattr(source, method)(target) + + assert result == target + assert not source.exists() + assert not source._local.exists() + assert not target._local.exists() + assert target.read_text() == "source content" + with source.open("x") as handle: + handle.write("recreated source") + assert source.read_text() == "recreated source" + + +@pytest.mark.parametrize("method", ["rename", "replace"]) +def test_move_to_same_path_preserves_cache(rig: CloudProviderTestRig, method): + cp = rig.create_cloud_path("same-path.txt") + cp.write_text("unchanged") + + assert getattr(cp, method)(cp) == cp + + assert cp._local.exists() + assert cp.read_text() == "unchanged" + + +def test_rmtree_clears_only_its_cache_subtree(rig: CloudProviderTestRig): + directory = rig.create_cloud_path("delete-tree/") + child = directory / "nested" / "file.txt" + sibling = rig.create_cloud_path("keep-tree/file.txt") + child.write_text("delete") + sibling.write_text("keep") + assert child._local.exists() + assert sibling._local.exists() + + directory.rmtree() + + assert not directory.exists() + assert not directory._local.exists() + assert sibling._local.exists() + assert sibling.read_text() == "keep" + + +def test_rmdir_clears_stale_cached_children(rig: CloudProviderTestRig, monkeypatch): + directory = rig.create_cloud_path("empty-directory/") + child = directory / "file.txt" + child.write_text("stale") + child.client._remove(child) + assert child._local.exists() + # Object stores need not retain an empty directory; some mocks cannot list it. + monkeypatch.setattr(directory, "iterdir", lambda: iter(())) + + directory.rmdir() + + assert not directory._local.exists() + + +@pytest.mark.parametrize("method", ["unlink", "rename"]) +def test_failed_mutation_preserves_source_cache(rig: CloudProviderTestRig, method, monkeypatch): + cp = rig.create_cloud_path("failed-mutation.txt") + cp.write_text("preserved") + + def fail(*args, **kwargs): + raise RuntimeError("simulated cloud failure") + + if method == "unlink": + monkeypatch.setattr(cp.client, "_remove", fail) + args = () + else: + monkeypatch.setattr(cp.client, "_move_file", fail) + args = (rig.create_cloud_path("failed-move-target.txt"),) + + with pytest.raises(RuntimeError, match="simulated cloud failure"): + getattr(cp, method)(*args) + + assert cp.exists() + assert cp._local.read_text() == "preserved" + + +@pytest.mark.parametrize("method", ["move", "move_into"]) +def test_native_move_invalidates_both_caches(rig: CloudProviderTestRig, method): + source = rig.create_cloud_path("native-move-source.txt") + target = rig.create_cloud_path( + "native-move-target/" if method == "move_into" else "native-move-target.txt" + ) + destination = target / source.name if method == "move_into" else target + source.write_text("source content") + destination.write_text("old target content") + # A stale destination cache must be invalidated even if its remote file is gone. + destination.client._remove(destination) + assert source.client is destination.client + assert source._local.exists() + assert destination._local.exists() + + result = getattr(source, method)(target, force_overwrite_to_cloud=True) + + assert result == destination + assert not source.exists() + assert not source._local.exists() + assert not destination._local.exists() + assert result.read_text() == "source content" + with source.open("x") as handle: + handle.write("recreated") + assert source.read_text() == "recreated" + + +@pytest.mark.parametrize("mode", ["x", "a"]) +def test_external_deletion_discards_clean_cache(rig: CloudProviderTestRig, mode): + cp = rig.create_cloud_path("externally-deleted.txt") + cp.write_text("old content") + cp.client._remove(cp) + assert cp._local.exists() + + with cp.open(mode) as handle: + handle.write("new content") + + assert cp.read_text() == "new content" + + +def test_refresh_preserves_missing_dirty_cache(rig: CloudProviderTestRig, monkeypatch): + cp = rig.create_cloud_path("dirty-refresh.txt") + + def fail_upload(*args, **kwargs): + raise RuntimeError("simulated upload failure") + + with monkeypatch.context() as patch: + patch.setattr(cp.client, "_upload_file", fail_upload) + with pytest.raises(RuntimeError, match="simulated upload failure"): + cp.write_text("pending local content") + assert cp._dirty + # The failed upload closed the buffer; avoid retrying it during fixture cleanup. + assert cp._handle.closed + cp._handle = None + + cp._refresh_cache() + + assert cp._local.read_text() == "pending local content" + + +def test_refresh_preserves_directory_cache(rig: CloudProviderTestRig, monkeypatch): + directory = rig.create_cloud_path("refresh-directory/") + child = directory / "file.txt" + child.write_text("keep cached child") + + def no_stat(): + raise NoStatError("directory has no file stat") + + monkeypatch.setattr(directory, "stat", no_stat) + directory._refresh_cache() + + assert child._local.read_text() == "keep cached child" + + +@pytest.mark.parametrize("method", ["rename", "replace"]) +def test_adls_directory_move_invalidates_nested_caches(azure_gen2_rig, method): + source = azure_gen2_rig.create_cloud_path("cached-directory") + source.mkdir() + (source / "nested").mkdir() + child = source / "nested" / "file.txt" + child.write_text("nested source content") + target = azure_gen2_rig.create_cloud_path("renamed-directory") + target._local.mkdir(parents=True, exist_ok=True) + (target._local / "stale.txt").write_text("stale destination cache") + assert child._local.exists() + + result = getattr(source, method)(target) + + assert result == target + assert not source.exists() + assert not source._local.exists() + assert not target._local.exists() + assert (target / "nested" / "file.txt").read_text() == "nested source content" + + +@pytest.mark.parametrize("method", ["move", "move_into"]) +def test_failed_native_move_preserves_both_caches(rig: CloudProviderTestRig, method, monkeypatch): + source = rig.create_cloud_path("failed-native-source.txt") + target = rig.create_cloud_path( + "failed-native-target/" if method == "move_into" else "failed-native-target.txt" + ) + destination = target / source.name if method == "move_into" else target + source.write_text("source content") + destination.write_text("target content") + + def fail(*args, **kwargs): + raise RuntimeError("simulated move failure") + + monkeypatch.setattr(source.client, "_move_file", fail) + with pytest.raises(RuntimeError, match="simulated move failure"): + getattr(source, method)(target, force_overwrite_to_cloud=True) + + assert source.exists() + assert destination.exists() + assert source._local.read_text() == "source content" + assert destination._local.read_text() == "target content" + + +def test_failed_adls_directory_move_preserves_caches(azure_gen2_rig, monkeypatch): + source = azure_gen2_rig.create_cloud_path("failed-directory") + source.mkdir() + child = source / "file.txt" + child.write_text("cached source content") + target = azure_gen2_rig.create_cloud_path("failed-directory-target") + target._local.mkdir(parents=True, exist_ok=True) + stale_target = target._local / "stale.txt" + stale_target.write_text("cached target content") + + def fail(*args, **kwargs): + raise RuntimeError("simulated directory move failure") + + monkeypatch.setattr(source.client, "_move_file", fail) + with pytest.raises(RuntimeError, match="simulated directory move failure"): + source.rename(target) + + assert source.exists() + assert child._local.read_text() == "cached source content" + assert stale_target.read_text() == "cached target content" + + +def test_adls_directory_self_rename_preserves_cache(azure_gen2_rig): + directory = azure_gen2_rig.create_cloud_path("same-directory") + directory.mkdir() + child = directory / "file.txt" + child.write_text("cached contents") + + assert directory.rename(directory) == directory + assert child._local.read_text() == "cached contents"