Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 66 additions & 10 deletions src/modelscope_hub/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

from __future__ import annotations

import fnmatch
import time
from collections.abc import Iterable, Mapping
from pathlib import Path
Expand Down Expand Up @@ -1669,25 +1670,30 @@ def delete_files(
self,
repo_id: str,
repo_type: RepoTypeLike,
file_paths: Iterable[str],
file_paths: Iterable[str] | str | None = None,
*,
delete_patterns: Iterable[str] | str | None = None,
commit_message: str | None = None,
revision: str | None = None,
) -> dict:
"""Delete one or more files from a repository in a single commit.
"""Delete repository files selected by paths or glob patterns.

The direct repository DELETE endpoints reject API-token authentication.
This method therefore sends commit ``delete`` actions through the same
supported write path as :meth:`upload_file` and :meth:`upload_folder`.
This method resolves optional glob patterns against the remote file list,
then sends commit ``delete`` actions through the same supported write
path as :meth:`upload_file` and :meth:`upload_folder`.

Parameters
----------
repo_id : str
Canonical ``owner/name`` identifier.
repo_type : str or RepoType
Repository type.
file_paths : iterable of str
Paths of files to remove. Empty entries are ignored.
file_paths : iterable of str or str, optional
Explicit repository-relative paths to remove.
delete_patterns : iterable of str or str, optional
Glob patterns matched against remote file paths. For example,
``"*.json"`` matches JSON files at any repository depth.
commit_message : str, optional
Message for the delete commit. Defaults to ``"Delete files"``.
revision : str, optional
Expand All @@ -1696,18 +1702,68 @@ def delete_files(
Returns
-------
dict
Summary with ``deleted_files`` and ``failed_files`` lists.
Summary with ``deleted_files``, ``failed_files``, and ``total_files``.
A pattern with no remote matches returns an empty successful summary.
"""
rt = self._normalize_repo_type(repo_type)
paths = [file_paths] if isinstance(file_paths, str) else file_paths
paths = self._normalize_delete_values(file_paths, "file_paths")
patterns = self._normalize_delete_values(
delete_patterns, "delete_patterns")
if not paths and not patterns:
raise InvalidParameter(
"Provide at least one file path or delete pattern.")

resolved_revision = revision or "master"
if patterns:
remote_paths = [
file.path
for file in self.list_repo_files(
repo_id,
rt,
revision=resolved_revision,
recursive=True,
)
if file.path and file.type != "tree"
]
paths.extend(
path for path in remote_paths
if any(fnmatch.fnmatchcase(path, pattern)
for pattern in patterns))

paths = list(dict.fromkeys(paths))
if not paths:
return {
"deleted_files": [],
"failed_files": [],
"total_files": 0,
}

return self.uploader.delete_files(
repo_id=repo_id,
repo_type=str(rt),
file_paths=[path for path in paths if path],
file_paths=paths,
commit_message=commit_message or "Delete files",
revision=revision or "master",
revision=resolved_revision,
)

@staticmethod
def _normalize_delete_values(
values: Iterable[str] | str | None,
parameter_name: str,
) -> list[str]:
"""Normalize a delete path or glob-pattern argument."""
if values is None:
return []
raw_values = [values] if isinstance(values, str) else list(values)
normalized: list[str] = []
for value in raw_values:
if not isinstance(value, str):
raise InvalidParameter(
f"{parameter_name} must contain only strings.")
if value:
normalized.append(value)
return normalized

# ==================================================================
# Versioning
# ==================================================================
Expand Down
30 changes: 30 additions & 0 deletions src/modelscope_hub/compat/hub_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,36 @@ def get_model_files(
result = [f for f in result if f["Path"] == prefix or str(f["Path"]).startswith(prefix + "/")]
return result

def delete_files(
self,
repo_id: str,
repo_type: str | RepoType = RepoType.MODEL,
delete_patterns: str | list[str] | None = None,
*,
file_paths: str | list[str] | None = None,
revision: str | None = DEFAULT_DATASET_REVISION,
commit_message: str | None = None,
endpoint: str | None = None,
token: str | None = None,
) -> dict:
"""Delete files selected by legacy glob patterns or explicit paths.

``delete_patterns`` preserves the historical ``modelscope.hub.api``
contract. Patterns are resolved against the remote repository by the
modelscope-hub facade before it commits atomic delete actions.
"""
api = self._api
if token or endpoint:
api = HubApi(endpoint=endpoint or self._endpoint, token=token)
return api.delete_files(
repo_id,
repo_type,
file_paths=file_paths,
delete_patterns=delete_patterns,
commit_message=commit_message,
revision=revision,
)

def create_repo(
self,
repo_id: str,
Expand Down
2 changes: 1 addition & 1 deletion src/modelscope_hub/version.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""Version information for modelscope_hub."""

__version__ = "0.4.1+main"
__version__ = "0.4.2+main"
54 changes: 54 additions & 0 deletions tests/test_compat_delete_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
from __future__ import annotations

from unittest import mock

from modelscope_hub.compat import LegacyHubApi
from modelscope_hub.constants import RepoType


def test_legacy_hub_api_preserves_delete_patterns_contract() -> None:
api = LegacyHubApi(endpoint="https://modelscope.cn", token="test-token")
expected = {
"deleted_files": ["config.json"],
"failed_files": [],
"total_files": 1,
}

with mock.patch.object(api._api, "delete_files", return_value=expected) as delete_files:
result = api.delete_files(
repo_id="owner/repo",
repo_type=RepoType.MODEL,
delete_patterns="*.json",
revision="main",
commit_message="Delete JSON files",
)

delete_files.assert_called_once_with(
"owner/repo",
RepoType.MODEL,
file_paths=None,
delete_patterns="*.json",
commit_message="Delete JSON files",
revision="main",
)
assert result == expected


def test_legacy_hub_api_delete_files_supports_explicit_paths() -> None:
api = LegacyHubApi(token="test-token")

with mock.patch.object(api._api, "delete_files", return_value={}) as delete_files:
api.delete_files(
"owner/repo",
repo_type="dataset",
file_paths=["data/old.json"],
)

delete_files.assert_called_once_with(
"owner/repo",
"dataset",
file_paths=["data/old.json"],
delete_patterns=None,
commit_message=None,
revision="master",
)
55 changes: 55 additions & 0 deletions tests/test_upload_lfs_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import hashlib
import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest
Expand Down Expand Up @@ -112,6 +113,60 @@ def test_hub_api_delete_files_delegates_to_upload_manager(
assert result == {"deleted_files": ["old.bin"]}


@pytest.mark.parametrize("repo_type", ["model", "dataset"])
def test_hub_api_delete_patterns_resolve_remote_paths(repo_type: str) -> None:
api = HubApi(token="test-token")
api._uploader = MagicMock()
api._uploader.delete_files.return_value = {
"deleted_files": ["config.json", "nested/metadata.json"],
"failed_files": [],
"total_files": 2,
}
api.list_repo_files = MagicMock(
return_value=[
SimpleNamespace(path="config.json", type="blob"),
SimpleNamespace(path="nested/metadata.json", type="blob"),
SimpleNamespace(path="weights.bin", type="blob"),
SimpleNamespace(path="nested", type="tree"),
])

result = api.delete_files(
"owner/repo",
repo_type,
delete_patterns="*.json",
commit_message="Remove JSON files",
revision="main",
)

api.list_repo_files.assert_called_once_with(
"owner/repo", repo_type, revision="main", recursive=True)
api._uploader.delete_files.assert_called_once_with(
repo_id="owner/repo",
repo_type=repo_type,
file_paths=["config.json", "nested/metadata.json"],
commit_message="Remove JSON files",
revision="main",
)
assert result["total_files"] == 2


def test_hub_api_delete_patterns_with_no_match_is_noop() -> None:
api = HubApi(token="test-token")
api._uploader = MagicMock()
api.list_repo_files = MagicMock(
return_value=[SimpleNamespace(path="weights.bin", type="blob")])

result = api.delete_files(
"owner/repo", "model", delete_patterns="*.json")

api._uploader.delete_files.assert_not_called()
assert result == {
"deleted_files": [],
"failed_files": [],
"total_files": 0,
}


def test_upload_file_normal_commits_inline_without_blob_api() -> None:
manager, client = _make_manager()

Expand Down
Loading