[#19463][feat] Add optional ModelScope model loading support

docs/source/llm-api/index.md

@@ -31,6 +31,38 @@ llm = LLM(model="TinyLlama/TinyLlama-1.1B-Chat-v1.0")
 
 You can also use [quantized checkpoints](https://huggingface.co/collections/nvidia/model-optimizer-66aa84f7966b3150262481a4) (FP4, FP8, etc) of popular models provided by NVIDIA in the same way.
 
+### Using a Model from ModelScope
+
+To resolve remote model IDs through [ModelScope](https://modelscope.cn/)
+instead of the Hugging Face Hub, install the optional client and enable the
+ModelScope download path before starting TensorRT-LLM:
+
+```console
+pip install 'modelscope>=1.20'
+export TRTLLM_USE_MODELSCOPE=true
+trtllm-serve Qwen/Qwen3-0.6B
+```
+
+The switch also applies to remote tokenizer and speculative-model IDs. Local
+paths are used as-is. Unset `TRTLLM_USE_MODELSCOPE`, or set it to `false`, to
+retain the default Hugging Face behavior.
+
+You can also install the client through the `tensorrt_llm[modelscope]` extra.
+ModelScope remains optional and is imported only when this switch is enabled.
+
+The integration routes TensorRT-LLM snapshot downloads through ModelScope and
+passes the resolved local directories to its tokenizer and configuration loaders.
+Explicit remote tokenizers download only tokenizer/configuration files, honoring
+`tokenizer_revision`. It deliberately does not call ModelScope's process-wide
+`patch_hub()`: unrelated Hugging Face consumers in the same process retain their
+own hub behavior. Applications that download additional assets outside these
+TensorRT-LLM paths must configure those consumers separately.
+
+The minimum client version, [ModelScope 1.20.0](https://github.com/modelscope/modelscope/blob/v1.20.0/modelscope/hub/snapshot_download.py),
+supports the `allow_patterns` and `ignore_patterns` glob arguments used here.
+The legacy `ignore_file_pattern` argument also interprets valid patterns as
+regular expressions; it is not interchangeable with the glob-only filter.
+
 ### 2. Using a Local Hugging Face Model
 
 To use a model from local storage, first download it manually:

setup.py

@@ -747,6 +747,7 @@ def get_build_state_options():
     scripts=['tensorrt_llm/llmapi/trtllm-llmapi-launch'],
     extras_require={
         "devel": devel_deps + grpc_smg_deps,
+        "modelscope": ["modelscope>=1.20"],
         "openengine": openengine_deps,
         "mx": mx_deps,
         "grpc-smg": grpc_smg_deps,

tensorrt_llm/llmapi/llm.py

@@ -1668,13 +1668,24 @@ def _build_model(self):
         self._engine_dir, self._hf_model_dir = model_loader()
 
     def _try_load_tokenizer(self) -> Optional[TokenizerBase]:
+        """Resolve the tokenizer for this LLM instance.
+
+        Prefers an explicitly supplied tokenizer, then a single LoRA
+        directory on the PyTorch backends, and otherwise falls back to the
+        downloaded model directory or the configured model reference.
+
+        Returns:
+            The resolved tokenizer, or None when tokenizer init is skipped.
+        """
         if self.args.skip_tokenizer_init:
             return None
 
         if self.args.tokenizer is not None:
             assert isinstance(self.args.tokenizer, TokenizerBase)
             return self.args.tokenizer
 
+        model_path = self._hf_model_dir or self.args.model
+
         # TODO smor- need to refine what is the desired behavior if lora is enabled
         # in terms of the tokenizer initialization process
         if hasattr(self.args, "backend") and self.args.backend in [
@@ -1689,15 +1700,15 @@ def _try_load_tokenizer(self) -> Optional[TokenizerBase]:
                         trust_remote_code=self.args.trust_remote_code,
                         use_fast=self.args.tokenizer_mode != 'slow')
                     if tokenizer is None:
-                        tokenizer_path = self.args.model
+                        tokenizer_path = model_path
                     else:
                         return tokenizer
                 except Exception:
-                    tokenizer_path = self.args.model
+                    tokenizer_path = model_path
             else:
-                tokenizer_path = self.args.model
+                tokenizer_path = model_path
         else:
-            tokenizer_path = self.args.model
+            tokenizer_path = model_path
         return ModelLoader.load_hf_tokenizer(
             tokenizer_path,
             trust_remote_code=self.args.trust_remote_code,
@@ -1716,7 +1727,16 @@ def tokenizer(self, tokenizer: TokenizerBase):
 
     def _try_load_generation_config(
             self) -> Optional[transformers.GenerationConfig]:
-        return ModelLoader.load_hf_generation_config(self.args.model)
+        """Load the Hugging Face generation config for this model.
+
+        Reads from the downloaded model directory when one is available so
+        that remotely fetched snapshots are not re-resolved.
+
+        Returns:
+            The generation config, or None when the model does not ship one.
+        """
+        model_dir = self._hf_model_dir or self.args.model
+        return ModelLoader.load_hf_generation_config(model_dir)
 
     def _try_load_generation_config_explicit_values(self) -> dict[str, Any]:
         if self.args.backend != "pytorch" or self.args.generation_config != "auto":
@@ -1726,8 +1746,17 @@ def _try_load_generation_config_explicit_values(self) -> dict[str, Any]:
 
     def _try_load_hf_model_config(
             self) -> Optional[transformers.PretrainedConfig]:
+        """Load the Hugging Face model config for this model.
+
+        Reads from the downloaded model directory when one is available so
+        that remotely fetched snapshots are not re-resolved.
+
+        Returns:
+            The model config, or None when the model does not ship one.
+        """
+        model_dir = self._hf_model_dir or self.args.model
         return ModelLoader.load_hf_model_config(
-            self.args.model, trust_remote_code=self.args.trust_remote_code)
+            model_dir, trust_remote_code=self.args.trust_remote_code)
 
     @set_api_status("prototype")
     def start_profile(self,

tensorrt_llm/llmapi/llm_args.py

@@ -75,8 +75,9 @@
 from ..usage.config import UsageContext  # noqa: F401
 from ..usage.config import TelemetryConfig, TelemetryField
 from .tokenizer import TokenizerBase, tokenizer_factory
-from .utils import (StrictBaseModel, generate_api_docs_as_docstring,
-                    get_type_repr)
+from .utils import (StrictBaseModel, download_hf_partial,
+                    generate_api_docs_as_docstring, get_type_repr,
+                    use_modelscope)
 
 TypeBaseModel = TypeVar("T", bound=BaseModel)
 
@@ -5267,6 +5268,14 @@ def validate_and_init_tokenizer(self):
 
             # Use tokenizer path if specified, otherwise use model path.
             load_path = self.tokenizer if self.tokenizer else self.model
+            if (use_modelscope() and isinstance(load_path, (str, Path))
+                    and not Path(load_path).exists()):
+                load_path = download_hf_partial(
+                    str(load_path), [
+                        "*.json", "*.jinja", "*.j2", "*.model", "*.py",
+                        "*.tiktoken", "*.txt"
+                    ],
+                    revision=self.tokenizer_revision)
             # The one loader for aliases and import paths; it raises
             # ValueError("Failed to load custom tokenizer ...") on failure.
             self.tokenizer = load_custom_tokenizer(
@@ -5275,6 +5284,14 @@ def validate_and_init_tokenizer(self):
                 trust_remote_code=self.trust_remote_code,
                 use_fast=self.tokenizer_mode != 'slow')
         else:
+            if (use_modelscope() and isinstance(self.tokenizer, (str, Path))
+                    and not Path(self.tokenizer).exists()):
+                self.tokenizer = download_hf_partial(
+                    str(self.tokenizer), [
+                        "*.json", "*.jinja", "*.j2", "*.model", "*.py",
+                        "*.tiktoken", "*.txt"
+                    ],
+                    revision=self.tokenizer_revision)
             self.tokenizer = tokenizer_factory(
                 self.tokenizer,
                 trust_remote_code=self.trust_remote_code,

tensorrt_llm/llmapi/utils.py

@@ -1,3 +1,5 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 import asyncio
 import collections
 import ctypes
@@ -27,7 +29,7 @@
 import huggingface_hub
 import psutil
 import torch
-from huggingface_hub import snapshot_download
+from huggingface_hub import snapshot_download as hf_snapshot_download
 from pydantic import BaseModel
 from tqdm.auto import tqdm
 
@@ -230,23 +232,30 @@ def __init__(self, *args, **kwargs):
 
 
 def download_hf_model(model: str, revision: Optional[str] = None) -> Path:
+    """Download a full model snapshot from the configured model hub.
+
+    Args:
+        model: The model name or path.
+        revision: The revision to use for the model.
+
+    Returns:
+        The path to the downloaded model.
+    """
     ignore_patterns = ["original/**/*"]
-    logger.info(f"Downloading model {model} from HuggingFace")
+    hub_name = "ModelScope" if use_modelscope() else "Hugging Face"
+    logger.info(f"Downloading model {model} from {hub_name}")
     with get_file_lock(model):
-        hf_folder = snapshot_download(
-            model,
-            local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
-            ignore_patterns=ignore_patterns,
-            revision=revision,
-            tqdm_class=DisabledTqdm)
-    logger.info(f"Finished downloading model {model} from HuggingFace")
-    return Path(hf_folder)
+        model_folder = _snapshot_download(model,
+                                          ignore_patterns=ignore_patterns,
+                                          revision=revision)
+    logger.info(f"Finished downloading model {model} from {hub_name}")
+    return Path(model_folder)
 
 
 def download_hf_partial(model: str,
                         allow_patterns: List[str],
                         revision: Optional[str] = None) -> Path:
-    """Download a partial model from HuggingFace.
+    """Download selected model files from the configured model hub.
 
     Args:
         model: The model name or path.
@@ -257,13 +266,54 @@ def download_hf_partial(model: str,
         The path to the downloaded model.
     """
     with get_file_lock(model):
-        hf_folder = snapshot_download(
-            model,
-            local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
-            revision=revision,
-            allow_patterns=allow_patterns,
-            tqdm_class=DisabledTqdm)
-    return Path(hf_folder)
+        model_folder = _snapshot_download(model,
+                                          revision=revision,
+                                          allow_patterns=allow_patterns)
+    return Path(model_folder)
+
+
+def use_modelscope() -> bool:
+    """Return whether remote model IDs should resolve through ModelScope."""
+    return os.environ.get("TRTLLM_USE_MODELSCOPE",
+                          "false").strip().lower() in ("1", "true")
+
+
+def _snapshot_download(model: str,
+                       revision: Optional[str] = None,
+                       ignore_patterns: Optional[List[str]] = None,
+                       allow_patterns: Optional[List[str]] = None) -> str:
+    """Download a snapshot from ModelScope or Hugging Face.
+
+    Keep the optional import in this boundary so standard TensorRT-LLM
+    installations do not need the ``modelscope`` package.
+    """
+    local_files_only = huggingface_hub.constants.HF_HUB_OFFLINE
+    if use_modelscope():
+        try:
+            from modelscope.hub.snapshot_download import snapshot_download
+        except ImportError as error:
+            raise ImportError(
+                "TRTLLM_USE_MODELSCOPE is enabled, but ModelScope is not "
+                "installed. Install it with `pip install 'modelscope>=1.20'`."
+            ) from error
+
+        kwargs = {
+            "model_id": model,
+            "local_files_only": local_files_only,
+            "revision": revision,
+        }
+        if ignore_patterns:
+            kwargs["ignore_patterns"] = ignore_patterns
+        if allow_patterns:
+            kwargs["allow_patterns"] = allow_patterns
+        return snapshot_download(**kwargs)
+
+    return hf_snapshot_download(model,
+                                local_files_only=local_files_only,
+                                ignore_patterns=ignore_patterns,
+                                allow_patterns=allow_patterns,
+                                revision=revision,
+                                tqdm_class=DisabledTqdm)
 
 
 def download_hf_pretrained_config(model: str,

tests/unittest/llmapi/test_utils.py

@@ -13,13 +13,106 @@
 from tensorrt_llm.llmapi.utils import (ApiStatusRegistry,
                                        _set_affinity_all_threads,
                                        configure_cpu_affinity,
+                                       download_hf_model, download_hf_partial,
                                        generate_api_docs_as_docstring)
 
 _TASK_DIR = "/proc/self/task"
 
 pytestmark = pytest.mark.cpu_only
 
 
+def _stub_modelscope(monkeypatch, snapshot_download):
+    """Register a fake ``modelscope`` package exposing ``snapshot_download``.
+
+    Args:
+        monkeypatch: The pytest monkeypatch fixture.
+        snapshot_download: The callable to install as the hub entry point.
+    """
+    modelscope = types.ModuleType("modelscope")
+    hub = types.ModuleType("modelscope.hub")
+    snapshot_module = types.ModuleType("modelscope.hub.snapshot_download")
+    snapshot_module.snapshot_download = snapshot_download
+    monkeypatch.setitem(sys.modules, "modelscope", modelscope)
+    monkeypatch.setitem(sys.modules, "modelscope.hub", hub)
+    monkeypatch.setitem(sys.modules, "modelscope.hub.snapshot_download",
+                        snapshot_module)
+
+
+def test_modelscope_download_maps_snapshot_filters(monkeypatch, tmp_path):
+    """Partial downloads forward allow patterns and revision to ModelScope."""
+    calls = []
+
+    def snapshot_download(**kwargs):
+        """Record the hub call and return the temporary snapshot path."""
+        calls.append(kwargs)
+        return str(tmp_path)
+
+    _stub_modelscope(monkeypatch, snapshot_download)
+    monkeypatch.setenv("TRTLLM_USE_MODELSCOPE", "true")
+    monkeypatch.setattr(llmapi_utils.huggingface_hub.constants,
+                        "HF_HUB_OFFLINE", True)
+
+    downloaded = download_hf_partial("Qwen/Qwen3-0.6B", ["*.json"],
+                                     revision="v1")
+
+    assert downloaded == tmp_path
+    assert calls == [{
+        "model_id": "Qwen/Qwen3-0.6B",
+        "local_files_only": True,
+        "revision": "v1",
+        "allow_patterns": ["*.json"],
+    }]
+
+
+def test_modelscope_download_maps_ignored_files(monkeypatch, tmp_path):
+    """Full downloads forward the default ignore patterns to ModelScope."""
+    calls = []
+
+    def snapshot_download(**kwargs):
+        """Record the hub call and return the temporary snapshot path."""
+        calls.append(kwargs)
+        return str(tmp_path)
+
+    _stub_modelscope(monkeypatch, snapshot_download)
+    monkeypatch.setenv("TRTLLM_USE_MODELSCOPE", "1")
+
+    downloaded = download_hf_model("Qwen/Qwen3-0.6B")
+
+    assert downloaded == tmp_path
+    assert calls[0]["ignore_patterns"] == ["original/**/*"]
+
+
+def test_hugging_face_download_remains_the_default(monkeypatch, tmp_path):
+    """Downloads route to Hugging Face when ModelScope is not enabled."""
+    calls = []
+
+    def snapshot_download(model, **kwargs):
+        """Record the hub call and return the temporary snapshot path."""
+        calls.append((model, kwargs))
+        return str(tmp_path)
+
+    monkeypatch.delenv("TRTLLM_USE_MODELSCOPE", raising=False)
+    monkeypatch.setattr(llmapi_utils, "hf_snapshot_download", snapshot_download)
+
+    downloaded = download_hf_partial("Qwen/Qwen3-0.6B", ["config.json"])
+
+    assert downloaded == tmp_path
+    assert calls[0][0] == "Qwen/Qwen3-0.6B"
+    assert calls[0][1]["allow_patterns"] == ["config.json"]
+
+
+def test_modelscope_download_requires_optional_dependency(monkeypatch):
+    """A missing ``modelscope`` install raises an actionable ImportError."""
+    monkeypatch.setenv("TRTLLM_USE_MODELSCOPE", "true")
+    monkeypatch.setitem(sys.modules, "modelscope", None)
+    monkeypatch.delitem(sys.modules,
+                        "modelscope.hub.snapshot_download",
+                        raising=False)
+
+    with pytest.raises(ImportError, match="modelscope>=1.20"):
+        download_hf_model("Qwen/Qwen3-0.6B")
+
+
 def test_api_status_registry():
 
     @ApiStatusRegistry.set_api_status("beta")