From ac9f222909e051215e05b3d645eb8a1b1beb495b Mon Sep 17 00:00:00 2001 From: zxy Date: Mon, 6 Jul 2026 12:00:30 +0800 Subject: [PATCH 1/5] fix: restrict remote media domains --- docs/zh_cn/multi_modal/multimodal_inputs.md | 6 + lmdeploy/api.py | 3 + lmdeploy/cli/serve.py | 7 ++ lmdeploy/pipeline.py | 3 + lmdeploy/serve/core/async_engine.py | 5 +- lmdeploy/serve/core/vl_async_engine.py | 5 +- lmdeploy/serve/openai/api_server.py | 4 + lmdeploy/serve/processors/multimodal.py | 37 ++++-- lmdeploy/vl/media/connection.py | 49 +++++--- tests/test_lmdeploy/test_content_merge.py | 59 +++++++++- tests/test_lmdeploy/test_vl/test_safe_url.py | 114 ++++++++++++++++++- 11 files changed, 259 insertions(+), 33 deletions(-) diff --git a/docs/zh_cn/multi_modal/multimodal_inputs.md b/docs/zh_cn/multi_modal/multimodal_inputs.md index d87b19e068..c25a4196fb 100644 --- a/docs/zh_cn/multi_modal/multimodal_inputs.md +++ b/docs/zh_cn/multi_modal/multimodal_inputs.md @@ -392,6 +392,12 @@ ______________________________________________________________________ ## 本地文件与 Base64 +当 API 服务面向不可信客户端时，建议在启动服务时使用 `--allowed-media-domains` 限制 HTTP(S) 媒体 URL 可访问的域名，降低 SSRF 风险。该参数按精确 hostname 匹配；例如 `--allowed-media-domains example.com` 只允许 `https://example.com/...`，不允许 `https://cdn.example.com/...`。 + +```shell +lmdeploy serve api_server --allowed-media-domains example.com +``` + 除 HTTP URL 外，lmdeploy 还支持： - **本地文件路径**，使用 `file://` 协议：`file:///absolute/path/to/file.jpg` diff --git a/lmdeploy/api.py b/lmdeploy/api.py index d674166ddf..a868ec5729 100644 --- a/lmdeploy/api.py +++ b/lmdeploy/api.py @@ -19,6 +19,7 @@ def pipeline(model_path: str, max_log_len: int | None = None, trust_remote_code: bool = False, speculative_config: SpeculativeConfig | None = None, + allowed_media_domains: list[str] | None = None, **kwargs): """Create a pipeline for inference. @@ -44,6 +45,7 @@ def pipeline(model_path: str, being printed in log. trust_remote_code: whether to trust remote code from model repositories. speculative_config: speculative decoding configuration. + allowed_media_domains: Optional HTTP(S) media URL domain allowlist. **kwargs: additional keyword arguments passed to the pipeline. Returns: @@ -77,6 +79,7 @@ def pipeline(model_path: str, max_log_len=max_log_len, trust_remote_code=trust_remote_code, speculative_config=speculative_config, + allowed_media_domains=allowed_media_domains, **kwargs) diff --git a/lmdeploy/cli/serve.py b/lmdeploy/cli/serve.py index 01ac1d44f1..8ef5764559 100644 --- a/lmdeploy/cli/serve.py +++ b/lmdeploy/cli/serve.py @@ -71,6 +71,11 @@ def add_parser_api_server(): 'engine’s tasks once the maximum number of concurrent requests is ' 'reached, regardless of any additional requests sent by clients ' 'concurrently during that time. Default to None.') + parser.add_argument('--allowed-media-domains', + nargs='+', + type=str, + default=None, + help='Exact hostnames allowed for HTTP(S) media URLs.') # common args ArgumentHelper.backend(parser) ArgumentHelper.log_level(parser) @@ -318,6 +323,7 @@ def api_server(args): reasoning_parser=args.reasoning_parser, tool_call_parser=args.tool_call_parser, speculative_config=speculative_config, + allowed_media_domains=args.allowed_media_domains, ) else: from lmdeploy.serve.openai.launch_server import launch_server @@ -350,6 +356,7 @@ def api_server(args): reasoning_parser=args.reasoning_parser, tool_call_parser=args.tool_call_parser, speculative_config=speculative_config, + allowed_media_domains=args.allowed_media_domains, ) @staticmethod diff --git a/lmdeploy/pipeline.py b/lmdeploy/pipeline.py index 23d80fd9db..8dcab8c614 100644 --- a/lmdeploy/pipeline.py +++ b/lmdeploy/pipeline.py @@ -40,6 +40,7 @@ def __init__(self, max_log_len: int | None = None, trust_remote_code: bool = False, speculative_config: SpeculativeConfig | None = None, + allowed_media_domains: list[str] | None = None, **kwargs): """Initialize Pipeline. @@ -51,6 +52,7 @@ def __init__(self, max_log_len: Max number of prompt characters or prompt tokens being printed in log. trust_remote_code: whether to trust remote code from model repositories. speculative_config: Speculative decoding configuration. + allowed_media_domains: Optional HTTP(S) media URL domain allowlist. **kwargs: Additional keyword arguments. """ @@ -82,6 +84,7 @@ def __init__(self, max_log_len=max_log_len, trust_remote_code=trust_remote_code, speculative_config=speculative_config, + allowed_media_domains=allowed_media_domains, **kwargs) self.internal_thread = _EventLoopThread(daemon=True) self.limiter: asyncio.Semaphore = None diff --git a/lmdeploy/serve/core/async_engine.py b/lmdeploy/serve/core/async_engine.py index 68d4c51f85..a7177a8f41 100644 --- a/lmdeploy/serve/core/async_engine.py +++ b/lmdeploy/serve/core/async_engine.py @@ -115,6 +115,7 @@ def __init__(self, max_log_len: int | None = None, trust_remote_code: bool = False, speculative_config: SpeculativeConfig | None = None, + allowed_media_domains: list[str] | None = None, **kwargs) -> None: logger.info(f'input backend={backend}, backend_config={backend_config}') logger.info(f'speculative_config={speculative_config}') @@ -123,7 +124,9 @@ def __init__(self, self.model_name = model_name if model_name else model_path self.chat_template = get_chat_template(model_path, chat_template_config, trust_remote_code=trust_remote_code) self.tokenizer = Tokenizer(model_path, trust_remote_code=trust_remote_code) - self.prompt_processor = MultimodalProcessor(self.tokenizer, self.chat_template) + self.prompt_processor = MultimodalProcessor(self.tokenizer, + self.chat_template, + allowed_media_domains=allowed_media_domains) self.hf_gen_cfg = get_hf_gen_cfg(model_path, trust_remote_code=trust_remote_code) self.arch, self.hf_cfg = get_model_arch(model_path, trust_remote_code=trust_remote_code) self.session_len = (_get_and_verify_max_len(self.hf_cfg, None) diff --git a/lmdeploy/serve/core/vl_async_engine.py b/lmdeploy/serve/core/vl_async_engine.py index d246a20f75..3de82c4583 100644 --- a/lmdeploy/serve/core/vl_async_engine.py +++ b/lmdeploy/serve/core/vl_async_engine.py @@ -18,6 +18,7 @@ def __init__(self, backend_config: TurbomindEngineConfig | PytorchEngineConfig | None = None, vision_config: VisionConfig | None = None, trust_remote_code: bool = False, + allowed_media_domains: list[str] | None = None, **kwargs) -> None: from lmdeploy.serve.processors import MultimodalProcessor from lmdeploy.utils import try_import_deeplink @@ -40,12 +41,14 @@ def __init__(self, backend=backend, backend_config=backend_config, trust_remote_code=trust_remote_code, + allowed_media_domains=allowed_media_domains, **kwargs) # Update prompt_processor to support multimodal processing self.prompt_processor = MultimodalProcessor(self.tokenizer, self.chat_template, vl_encoder=self.vl_encoder, - backend=backend) + backend=backend, + allowed_media_domains=allowed_media_domains) if self.model_name == 'base': raise RuntimeError( 'please specify chat template as guided in https://lmdeploy.readthedocs.io/en/latest/inference/vl_pipeline.html#set-chat-template' # noqa: E501 diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index 892faa5692..ee7ce0b7ae 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -1547,6 +1547,7 @@ def serve(model_path: str, allow_terminate_by_client: bool = False, enable_abort_handling: bool = False, speculative_config: SpeculativeConfig | None = None, + allowed_media_domains: list[str] | None = None, **kwargs): """An example to perform model inference through the command line interface. @@ -1599,6 +1600,8 @@ def serve(model_path: str, reasoning_parser (str): The reasoning parser name. tool_call_parser (str): The tool call parser name. allow_terminate_by_client (bool): Allow request from client to terminate server. + allowed_media_domains (list[str] | None): Optional exact hostname + allowlist for HTTP(S) media URLs. """ if os.getenv('TM_LOG_LEVEL') is None: os.environ['TM_LOG_LEVEL'] = log_level @@ -1631,6 +1634,7 @@ def serve(model_path: str, max_log_len=max_log_len, trust_remote_code=trust_remote_code, speculative_config=speculative_config, + allowed_media_domains=allowed_media_domains, **kwargs) set_parsers(reasoning_parser, tool_call_parser) diff --git a/lmdeploy/serve/processors/multimodal.py b/lmdeploy/serve/processors/multimodal.py index cf2452935e..01cca5baad 100644 --- a/lmdeploy/serve/processors/multimodal.py +++ b/lmdeploy/serve/processors/multimodal.py @@ -25,7 +25,8 @@ def __init__(self, tokenizer: Tokenizer, chat_template: BaseChatTemplate, vl_encoder=None, - backend: str | None = None): + backend: str | None = None, + allowed_media_domains: list[str] | None = None): """Initialize MultimodalProcessor. Args: @@ -33,11 +34,13 @@ def __init__(self, chat_template: Chat template instance for message processing. vl_encoder: Optional ImageEncoder instance for multimodal processing. backend: Optional backend name ('turbomind' or 'pytorch') for multimodal processing. + allowed_media_domains: Optional HTTP(S) media URL domain allowlist. """ self.tokenizer = tokenizer self.chat_template = chat_template self.vl_encoder = vl_encoder self.backend = backend + self.allowed_media_domains = allowed_media_domains @staticmethod def merge_message_content(msg: dict) -> dict: @@ -91,8 +94,11 @@ def merge_message_content(msg: dict) -> dict: return result @staticmethod - def _parse_multimodal_item(i: int, in_messages: list[dict], out_messages: list[dict], media_io_kwargs: dict[str, - Any]): + def _parse_multimodal_item(i: int, + in_messages: list[dict], + out_messages: list[dict], + media_io_kwargs: dict[str, Any], + allowed_media_domains: list[str] | None = None): """Synchronous helper to parse a single multimodal message item.""" role = in_messages[i]['role'] content = in_messages[i]['content'] @@ -140,21 +146,29 @@ def _require_data_src(): if isinstance(data_src, PIL.Image.Image): data = data_src elif isinstance(data_src, str): - data = load_from_url(data_src, ImageMediaIO(**media_io_kwargs.get('image', {}))) + data = load_from_url(data_src, + ImageMediaIO(**media_io_kwargs.get('image', {})), + allowed_media_domains=allowed_media_domains) else: raise ValueError(f'Invalid multimodal image item at index {i}: {item}. ' 'Expected a str URL/path/data URL or PIL.Image.Image.') elif item_type in ('video_url', 'video'): modality = Modality.VIDEO data, metadata = load_from_url( - _require_data_src(), VideoMediaIO(image_io=ImageMediaIO(), **media_io_kwargs.get('video', {}))) + _require_data_src(), + VideoMediaIO(image_io=ImageMediaIO(), **media_io_kwargs.get('video', {})), + allowed_media_domains=allowed_media_domains) item_params['video_metadata'] = metadata elif item_type in ('audio_url', 'audio'): modality = Modality.AUDIO - data = load_from_url(_require_data_src(), AudioMediaIO(**media_io_kwargs.get('audio', {}))) + data = load_from_url(_require_data_src(), + AudioMediaIO(**media_io_kwargs.get('audio', {})), + allowed_media_domains=allowed_media_domains) elif item_type in ('time_series_url', 'time_series'): modality = Modality.TIME_SERIES - data = load_from_url(_require_data_src(), TimeSeriesMediaIO(**media_io_kwargs.get('time_series', {}))) + data = load_from_url(_require_data_src(), + TimeSeriesMediaIO(**media_io_kwargs.get('time_series', {})), + allowed_media_domains=allowed_media_domains) else: raise NotImplementedError(f'unknown type: {item_type}') @@ -164,7 +178,8 @@ def _require_data_src(): @staticmethod async def async_parse_multimodal_item(messages: list[dict], - media_io_kwargs: dict[str, Any] | None = None) -> list[dict]: + media_io_kwargs: dict[str, Any] | None = None, + allowed_media_domains: list[str] | None = None) -> list[dict]: """Convert user-input multimodal data into GPT4V message format.""" if isinstance(messages, dict): messages = [messages] @@ -176,7 +191,7 @@ async def async_parse_multimodal_item(messages: list[dict], await asyncio.gather(*[ loop.run_in_executor(None, MultimodalProcessor._parse_multimodal_item, i, messages, out_messages, - media_io_kwargs) for i in range(len(messages)) + media_io_kwargs, allowed_media_domains) for i in range(len(messages)) ]) return out_messages @@ -386,7 +401,9 @@ async def _get_multimodal_prompt_input(self, """Process multimodal prompt and return processed data for inference engines.""" chat_template = self.chat_template if do_preprocess else BaseChatTemplate() - messages = await self.async_parse_multimodal_item(messages, media_io_kwargs) + messages = await self.async_parse_multimodal_item(messages, + media_io_kwargs, + allowed_media_domains=self.allowed_media_domains) if self.backend == 'turbomind': if self.vl_encoder._uses_new_preprocess: diff --git a/lmdeploy/vl/media/connection.py b/lmdeploy/vl/media/connection.py index 6999761dd4..6a897a87bf 100644 --- a/lmdeploy/vl/media/connection.py +++ b/lmdeploy/vl/media/connection.py @@ -4,10 +4,11 @@ import socket from pathlib import Path from typing import TypeVar -from urllib.parse import ParseResult, urlparse +from urllib.parse import ParseResult, urljoin, urlparse from urllib.request import url2pathname import requests +from urllib3.util import parse_url from .base import MediaIO from .image import ImageMediaIO @@ -25,13 +26,15 @@ def _is_safe_url(url: str) -> tuple[bool, str]: """Check if the URL is safe to fetch (not internal/private).""" try: - parsed = urlparse(url) + parsed = parse_url(url) if parsed.scheme not in ('http', 'https'): return False, f'Unsupported scheme: {parsed.scheme}' hostname = parsed.hostname if not hostname: return False, 'Could not parse hostname from URL' + if hostname.startswith('[') and hostname.endswith(']'): + hostname = hostname[1:-1] # check all IPs (IPv4 + IPv6) using getaddrinfo try: @@ -51,12 +54,9 @@ def _is_safe_url(url: str) -> tuple[bool, str]: return False, f'URL validation failed: {str(e)}' -def _load_http_url(url_spec: ParseResult, media_io: MediaIO[_M]) -> _M: +def _load_http_url(url_spec: ParseResult, media_io: MediaIO[_M], + allowed_media_domains: list[str] | None = None) -> _M: url = url_spec.geturl() - is_safe, reason = _is_safe_url(url) - if not is_safe: - raise ValueError(f'URL is blocked for security reasons: {reason}') - fetch_timeout = 10 if isinstance(media_io, ImageMediaIO): fetch_timeout = int(os.environ.get('LMDEPLOY_IMAGE_FETCH_TIMEOUT', 10)) @@ -64,11 +64,34 @@ def _load_http_url(url_spec: ParseResult, media_io: MediaIO[_M]) -> _M: fetch_timeout = int(os.environ.get('LMDEPLOY_VIDEO_FETCH_TIMEOUT', 30)) client = requests.Session() - client.max_redirects = 3 - response = client.get(url_spec.geturl(), headers=headers, timeout=fetch_timeout, allow_redirects=True) - response.raise_for_status() + max_redirects = 3 + for _ in range(max_redirects + 1): + parsed = parse_url(url) + if parsed.scheme not in ('http', 'https'): + raise ValueError(f'URL is blocked for security reasons: Unsupported scheme: {parsed.scheme}') + if not parsed.hostname: + raise ValueError('URL is blocked for security reasons: Could not parse hostname from URL') + if allowed_media_domains and parsed.hostname not in allowed_media_domains: + raise ValueError('The URL must be from one of the allowed domains: ' + f'{allowed_media_domains}. Input URL domain: {parsed.hostname}') + + is_safe, reason = _is_safe_url(parsed.url) + if not is_safe: + raise ValueError(f'URL is blocked for security reasons: {reason}') + + response = client.get(parsed.url, headers=headers, timeout=fetch_timeout, allow_redirects=False) + + if 300 <= response.status_code < 400: + location = response.headers.get('Location') + if not location: + raise ValueError('Redirect response missing Location header') + url = urljoin(parsed.url, location) + continue + + response.raise_for_status() + return media_io.load_bytes(response.content) - return media_io.load_bytes(response.content) + raise ValueError('Exceeded maximum media URL redirects') def _load_data_url(url_spec: ParseResult, media_io: MediaIO[_M]) -> _M: @@ -92,12 +115,12 @@ def _load_file_url(url_spec: ParseResult, media_io: MediaIO[_M]) -> _M: return media_io.load_file(filepath) -def load_from_url(url: str, media_io: MediaIO[_M]) -> _M: +def load_from_url(url: str, media_io: MediaIO[_M], allowed_media_domains: list[str] | None = None) -> _M: """Load media from a HTTP, data or file url.""" url_spec = urlparse(url) if url_spec.scheme and url_spec.scheme.startswith('http'): - return _load_http_url(url_spec, media_io) + return _load_http_url(url_spec, media_io, allowed_media_domains) if url_spec.scheme == 'data': return _load_data_url(url_spec, media_io) diff --git a/tests/test_lmdeploy/test_content_merge.py b/tests/test_lmdeploy/test_content_merge.py index d494a8fcbd..2cc008a611 100644 --- a/tests/test_lmdeploy/test_content_merge.py +++ b/tests/test_lmdeploy/test_content_merge.py @@ -1,4 +1,3 @@ -import asyncio import sys import pytest @@ -242,7 +241,7 @@ class FakeAudioMediaIO: def __init__(self, **kwargs): self.kwargs = kwargs - def fake_load_from_url(data_src, media_io): + def fake_load_from_url(data_src, media_io, allowed_media_domains=None): load_calls.append((data_src, type(media_io).__name__)) if isinstance(media_io, FakeVideoMediaIO): return f'loaded:{data_src}', {'duration': 2} @@ -304,7 +303,8 @@ def fake_load_from_url(data_src, media_io): ] }] - parsed = asyncio.run(MultimodalProcessor.async_parse_multimodal_item(messages)) + parsed = [None] * len(messages) + MultimodalProcessor._parse_multimodal_item(0, messages, parsed, {}) content = parsed[0]['content'] assert content[0] == {'type': 'text', 'text': 'describe'} @@ -333,22 +333,71 @@ def fake_load_from_url(data_src, media_io): ] +def test_async_parse_multimodal_item_passes_allowed_media_domains(monkeypatch): + """Test server-owned domain allowlist is forwarded to URL loaders.""" + load_calls = [] + + class FakeVideoMediaIO: + + def __init__(self, image_io=None, **kwargs): + self.image_io = image_io + self.kwargs = kwargs + + def fake_load_from_url(data_src, media_io, allowed_media_domains=None): + load_calls.append((data_src, type(media_io).__name__, allowed_media_domains)) + if isinstance(media_io, FakeVideoMediaIO): + return f'loaded:{data_src}', {'duration': 2} + return f'loaded:{data_src}' + + monkeypatch.setattr(multimodal_module, 'VideoMediaIO', FakeVideoMediaIO) + monkeypatch.setattr(multimodal_module, 'load_from_url', fake_load_from_url) + + messages = [{ + 'role': + 'user', + 'content': [ + { + 'type': 'image_url', + 'image_url': { + 'url': 'https://example.com/a.png', + } + }, + { + 'type': 'video_url', + 'video_url': { + 'url': 'https://example.com/a.mp4', + } + }, + ] + }] + + out_messages = [None] * len(messages) + MultimodalProcessor._parse_multimodal_item(0, messages, out_messages, {}, ['example.com']) + + assert load_calls == [ + ('https://example.com/a.png', 'ImageMediaIO', ['example.com']), + ('https://example.com/a.mp4', 'FakeVideoMediaIO', ['example.com']), + ] + + @pytest.mark.parametrize('item', [{'type': 'image_url'}, {'type': 'image', 'image': {}}, {'type': 'time_series', 'time_series': {'sr': 16000}}]) def test_async_parse_multimodal_item_rejects_missing_payload(item): """Test missing multimodal payloads fail with a clear error.""" messages = [{'role': 'user', 'content': [item]}] + out_messages = [None] * len(messages) with pytest.raises(ValueError, match='Expected .* direct value or a dict containing "url" or "data"'): - asyncio.run(MultimodalProcessor.async_parse_multimodal_item(messages)) + MultimodalProcessor._parse_multimodal_item(0, messages, out_messages, {}) def test_async_parse_multimodal_item_rejects_unknown_type(): """Test unknown multimodal item types still fail explicitly.""" messages = [{'role': 'user', 'content': [{'type': 'unknown_media', 'unknown_media': 'file:///tmp/a.bin'}]}] + out_messages = [None] * len(messages) with pytest.raises(NotImplementedError, match='unknown type: unknown_media'): - asyncio.run(MultimodalProcessor.async_parse_multimodal_item(messages)) + MultimodalProcessor._parse_multimodal_item(0, messages, out_messages, {}) def test_has_multimodal_input_detects_all_supported_types(): diff --git a/tests/test_lmdeploy/test_vl/test_safe_url.py b/tests/test_lmdeploy/test_vl/test_safe_url.py index f919b3f0b9..8b8a9b7b19 100644 --- a/tests/test_lmdeploy/test_vl/test_safe_url.py +++ b/tests/test_lmdeploy/test_vl/test_safe_url.py @@ -40,8 +40,116 @@ def test_load_http_url_logic(mock_safe, mock_get): media_io = MagicMock() url_spec = urlparse('https://example.com/img.jpg') - # test success with allow_redirects=True - mock_get.return_value = MagicMock(content=b'data', status_code=200) + # test success with manual redirect handling disabled for the request + mock_get.return_value = MagicMock(content=b'data', status_code=200, is_redirect=False) media_io.load_bytes.return_value = 'loaded' assert _load_http_url(url_spec, media_io) == 'loaded' - assert mock_get.call_args.kwargs['allow_redirects'] is True + assert mock_get.call_args.args == ('https://example.com/img.jpg', ) + assert mock_get.call_args.kwargs['allow_redirects'] is False + + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_allowed_media_domains_exact_match(mock_safe, mock_get): + media_io = MagicMock() + media_io.load_bytes.return_value = 'loaded' + mock_get.return_value = MagicMock(content=b'data', status_code=200, is_redirect=False) + + assert _load_http_url(urlparse('https://example.com/img.jpg'), media_io, + allowed_media_domains=['example.com']) == 'loaded' + + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_allowed_media_domains_rejects_subdomain(mock_safe, mock_get): + media_io = MagicMock() + + with pytest.raises(ValueError, match='allowed domains'): + _load_http_url(urlparse('https://cdn.example.com/img.jpg'), media_io, allowed_media_domains=['example.com']) + + mock_get.assert_not_called() + + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_rejects_backslash_host_confusion(mock_safe, mock_get): + media_io = MagicMock() + + with pytest.raises(ValueError, match='allowed domains'): + _load_http_url(urlparse(r'https://example.com\@safe.example.org/img.jpg'), + media_io, + allowed_media_domains=['safe.example.org']) + + mock_get.assert_not_called() + + +@patch('requests.Session.get') +def test_load_http_url_allowed_domain_still_blocks_non_global_ip(mock_get): + media_io = MagicMock() + + with patch('socket.getaddrinfo', return_value=[(socket.AF_INET, None, None, None, ('127.0.0.1', 80))]): + with pytest.raises(ValueError, match='Blocked non-global IP detected'): + _load_http_url(urlparse('http://localhost/img.jpg'), media_io, allowed_media_domains=['localhost']) + + mock_get.assert_not_called() + + +def _mock_response(content=b'data', *, redirect_location=None): + response = MagicMock(content=content, status_code=302 if redirect_location is not None else 200) + response.is_redirect = redirect_location is not None + response.headers = {} + if redirect_location is not None: + response.headers['Location'] = redirect_location + return response + + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_validates_allowed_domain_after_redirect(mock_safe, mock_get): + media_io = MagicMock() + media_io.load_bytes.return_value = 'loaded' + mock_get.side_effect = [ + _mock_response(redirect_location='https://example.com/final.jpg'), + _mock_response(content=b'final'), + ] + + assert _load_http_url(urlparse('https://example.com/img.jpg'), media_io, + allowed_media_domains=['example.com']) == 'loaded' + assert mock_get.call_count == 2 + assert mock_get.call_args_list[1].args == ('https://example.com/final.jpg', ) + + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_blocks_disallowed_domain_after_redirect(mock_safe, mock_get): + media_io = MagicMock() + mock_get.return_value = _mock_response(redirect_location='https://evil.example/final.jpg') + + with pytest.raises(ValueError, match='allowed domains'): + _load_http_url(urlparse('https://example.com/img.jpg'), media_io, allowed_media_domains=['example.com']) + + assert mock_get.call_count == 1 + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_rejects_redirect_without_location(mock_safe, mock_get): + media_io = MagicMock() + mock_get.return_value = MagicMock(content=b'', status_code=302, headers={}) + + with pytest.raises(ValueError, match='Redirect response missing Location header'): + _load_http_url(urlparse('https://example.com/img.jpg'), media_io) + + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', + side_effect=[(True, ''), (False, 'Blocked non-global IP detected: 127.0.0.1')]) +def test_load_http_url_blocks_unsafe_url_after_redirect(mock_safe, mock_get): + media_io = MagicMock() + mock_get.return_value = _mock_response(redirect_location='http://127.0.0.1/final.jpg') + + with pytest.raises(ValueError, match='Blocked non-global IP detected'): + _load_http_url(urlparse('https://example.com/img.jpg'), + media_io, + allowed_media_domains=['example.com', '127.0.0.1']) + + assert mock_get.call_count == 1 From 6b1d113df1f9bd0f5bcd658a4328bdfe145c27cb Mon Sep 17 00:00:00 2001 From: zxy Date: Mon, 6 Jul 2026 12:28:22 +0800 Subject: [PATCH 2/5] test: address media url review feedback --- lmdeploy/vl/media/connection.py | 2 +- tests/test_lmdeploy/test_content_merge.py | 25 +++++++++++--------- tests/test_lmdeploy/test_vl/test_safe_url.py | 19 ++++++++++++--- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/lmdeploy/vl/media/connection.py b/lmdeploy/vl/media/connection.py index 6a897a87bf..2ff1a6e496 100644 --- a/lmdeploy/vl/media/connection.py +++ b/lmdeploy/vl/media/connection.py @@ -81,7 +81,7 @@ def _load_http_url(url_spec: ParseResult, media_io: MediaIO[_M], response = client.get(parsed.url, headers=headers, timeout=fetch_timeout, allow_redirects=False) - if 300 <= response.status_code < 400: + if response.is_redirect: location = response.headers.get('Location') if not location: raise ValueError('Redirect response missing Location header') diff --git a/tests/test_lmdeploy/test_content_merge.py b/tests/test_lmdeploy/test_content_merge.py index cc9023385e..5b50706ed9 100644 --- a/tests/test_lmdeploy/test_content_merge.py +++ b/tests/test_lmdeploy/test_content_merge.py @@ -1,3 +1,4 @@ +import asyncio import sys import pytest @@ -9,6 +10,14 @@ multimodal_module = sys.modules[MultimodalProcessor.__module__] +def _run_async(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + class TestMergeMessageContent: """Test suite for merge_message_content function.""" @@ -303,8 +312,7 @@ def fake_load_from_url(data_src, media_io, allowed_media_domains=None): ] }] - parsed = [None] * len(messages) - MultimodalProcessor._parse_multimodal_item(0, messages, parsed, {}) + parsed = _run_async(MultimodalProcessor.async_parse_multimodal_item(messages)) content = parsed[0]['content'] assert content[0] == {'type': 'text', 'text': 'describe'} @@ -371,8 +379,7 @@ def fake_load_from_url(data_src, media_io, allowed_media_domains=None): ] }] - out_messages = [None] * len(messages) - MultimodalProcessor._parse_multimodal_item(0, messages, out_messages, {}, ['example.com']) + _run_async(MultimodalProcessor.async_parse_multimodal_item(messages, allowed_media_domains=['example.com'])) assert load_calls == [ ('https://example.com/a.png', 'ImageMediaIO', ['example.com']), @@ -429,9 +436,7 @@ def fake_load_from_url(data_src, media_io, allowed_media_domains=None): }, ] - parsed = [None] * len(messages) - for i in range(len(messages)): - MultimodalProcessor._parse_multimodal_item(i, messages, parsed, {}) + parsed = _run_async(MultimodalProcessor.async_parse_multimodal_item(messages)) assert len(parsed) == 3 assert parsed[2]['role'] == 'tool' @@ -451,19 +456,17 @@ def fake_load_from_url(data_src, media_io, allowed_media_domains=None): def test_async_parse_multimodal_item_rejects_missing_payload(item): """Test missing multimodal payloads fail with a clear error.""" messages = [{'role': 'user', 'content': [item]}] - out_messages = [None] * len(messages) with pytest.raises(ValueError, match='Expected .* direct value or a dict containing "url" or "data"'): - MultimodalProcessor._parse_multimodal_item(0, messages, out_messages, {}) + _run_async(MultimodalProcessor.async_parse_multimodal_item(messages)) def test_async_parse_multimodal_item_rejects_unknown_type(): """Test unknown multimodal item types still fail explicitly.""" messages = [{'role': 'user', 'content': [{'type': 'unknown_media', 'unknown_media': 'file:///tmp/a.bin'}]}] - out_messages = [None] * len(messages) with pytest.raises(NotImplementedError, match='unknown type: unknown_media'): - MultimodalProcessor._parse_multimodal_item(0, messages, out_messages, {}) + _run_async(MultimodalProcessor.async_parse_multimodal_item(messages)) def test_has_multimodal_input_detects_all_supported_types(): diff --git a/tests/test_lmdeploy/test_vl/test_safe_url.py b/tests/test_lmdeploy/test_vl/test_safe_url.py index 8b8a9b7b19..828bfc4673 100644 --- a/tests/test_lmdeploy/test_vl/test_safe_url.py +++ b/tests/test_lmdeploy/test_vl/test_safe_url.py @@ -94,8 +94,9 @@ def test_load_http_url_allowed_domain_still_blocks_non_global_ip(mock_get): mock_get.assert_not_called() -def _mock_response(content=b'data', *, redirect_location=None): - response = MagicMock(content=content, status_code=302 if redirect_location is not None else 200) +def _mock_response(content=b'data', *, redirect_location=None, status_code=None): + status_code = status_code if status_code is not None else (302 if redirect_location is not None else 200) + response = MagicMock(content=content, status_code=status_code) response.is_redirect = redirect_location is not None response.headers = {} if redirect_location is not None: @@ -130,11 +131,23 @@ def test_load_http_url_blocks_disallowed_domain_after_redirect(mock_safe, mock_g assert mock_get.call_count == 1 + +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_does_not_treat_all_3xx_as_redirect(mock_safe, mock_get): + media_io = MagicMock() + media_io.load_bytes.return_value = 'loaded' + mock_get.return_value = _mock_response(content=b'cached', status_code=304) + + assert _load_http_url(urlparse('https://example.com/img.jpg'), media_io) == 'loaded' + media_io.load_bytes.assert_called_once_with(b'cached') + + @patch('requests.Session.get') @patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) def test_load_http_url_rejects_redirect_without_location(mock_safe, mock_get): media_io = MagicMock() - mock_get.return_value = MagicMock(content=b'', status_code=302, headers={}) + mock_get.return_value = MagicMock(content=b'', status_code=302, is_redirect=True, headers={}) with pytest.raises(ValueError, match='Redirect response missing Location header'): _load_http_url(urlparse('https://example.com/img.jpg'), media_io) From 7be9bd151aee5f2d02c2698ac3d84e87841a20a4 Mon Sep 17 00:00:00 2001 From: zxy Date: Fri, 10 Jul 2026 16:36:51 +0800 Subject: [PATCH 3/5] docs: add english media domain guidance --- docs/en/multi_modal/multimodal_inputs.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/en/multi_modal/multimodal_inputs.md b/docs/en/multi_modal/multimodal_inputs.md index 4f78ad2504..e19205cf95 100644 --- a/docs/en/multi_modal/multimodal_inputs.md +++ b/docs/en/multi_modal/multimodal_inputs.md @@ -393,6 +393,12 @@ ______________________________________________________________________ ## Local Files and Base64 +When the API server is exposed to untrusted clients, use `--allowed-media-domains` at startup to restrict which hostnames HTTP(S) media URLs can fetch from and reduce SSRF risk. The option matches exact hostnames; for example, `--allowed-media-domains example.com` allows `https://example.com/...` but not `https://cdn.example.com/...`. + +```shell +lmdeploy serve api_server --allowed-media-domains example.com +``` + In addition to HTTP URLs, lmdeploy accepts: - **Local file paths** via `file://` scheme: `file:///absolute/path/to/file.jpg` From 43573e2ebc7cb37ddf8ca3f1e9df6eead64592fa Mon Sep 17 00:00:00 2001 From: zxy Date: Fri, 10 Jul 2026 16:42:36 +0800 Subject: [PATCH 4/5] fix: honor media domains for tuple prompts --- lmdeploy/pipeline.py | 7 ++++--- lmdeploy/serve/processors/multimodal.py | 10 ++++------ tests/test_lmdeploy/test_content_merge.py | 20 ++++++++++++++++++++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/lmdeploy/pipeline.py b/lmdeploy/pipeline.py index 8dcab8c614..2f22097764 100644 --- a/lmdeploy/pipeline.py +++ b/lmdeploy/pipeline.py @@ -90,6 +90,7 @@ def __init__(self, self.limiter: asyncio.Semaphore = None self.session_mgr = self.async_engine.session_mgr self.backend_config = self.async_engine.backend_config + self.allowed_media_domains = allowed_media_domains self.async_engine.start_loop(self.internal_thread.loop, use_async_api=False) def infer(self, @@ -115,7 +116,7 @@ def infer(self, """ is_single = self._is_single(prompts) # format prompts to openai message format, which is a list of dicts - prompts = MultimodalProcessor.format_prompts(prompts) + prompts = MultimodalProcessor.format_prompts(prompts, allowed_media_domains=self.allowed_media_domains) pbar = tqdm.tqdm(total=len(prompts)) if use_tqdm else None outputs = [] try: @@ -165,7 +166,7 @@ def stream_infer(self, Returns: Iterator: A generator that yields the output (i.e. instance of class ``Response``) of the inference. """ - prompts = MultimodalProcessor.format_prompts(prompts) + prompts = MultimodalProcessor.format_prompts(prompts, allowed_media_domains=self.allowed_media_domains) requests = self._request_generator(prompts, sessions=sessions, gen_config=gen_config, @@ -204,7 +205,7 @@ def chat(self, session = self.session_mgr.get() session.update(prompt=prompt, response=None) - prompt = MultimodalProcessor.format_prompts(prompt) + prompt = MultimodalProcessor.format_prompts(prompt, allowed_media_domains=self.allowed_media_domains) sequence_start = session.step == 0 generator = self.stream_infer(prompts=prompt, diff --git a/lmdeploy/serve/processors/multimodal.py b/lmdeploy/serve/processors/multimodal.py index d663d3c59c..45c8f68fde 100644 --- a/lmdeploy/serve/processors/multimodal.py +++ b/lmdeploy/serve/processors/multimodal.py @@ -281,7 +281,7 @@ async def get_prompt_input(self, raise RuntimeError(f'unsupported prompt type: {type(prompt)}') @staticmethod - def format_prompts(prompts: Any) -> list[dict]: + def format_prompts(prompts: Any, allowed_media_domains: list[str] | None = None) -> list[dict]: """Format prompts.""" if not isinstance(prompts, list): prompts = [prompts] @@ -294,7 +294,7 @@ def format_prompts(prompts: Any) -> list[dict]: if all(MultimodalProcessor._is_str_images_pair(prompt) for prompt in prompts): # batch of (prompt, image or [images]) or (image or [images], prompt) -> # [[openai_gpt4v_message], [openai_gpt4v_message], ...] - return [[MultimodalProcessor._re_format_prompt_images_pair(prompt)] for prompt in prompts] + return [[MultimodalProcessor._re_format_prompt_images_pair(prompt, allowed_media_domains)] for prompt in prompts] raise ValueError(f'Unsupported prompts: {prompts}. Only support str, openai message format, ' 'or (prompt, image or [images]) or (image or [images], prompt) pair.') @@ -325,10 +325,8 @@ def _is_image_list(obj) -> bool: return isinstance(obj, list) and all(MultimodalProcessor._is_image(img) for img in obj) @staticmethod - def _re_format_prompt_images_pair(prompt: tuple) -> dict: + def _re_format_prompt_images_pair(prompt: tuple, allowed_media_domains: list[str] | None = None) -> dict: """Reformat the prompt to openai message format.""" - from lmdeploy.vl import load_image - messages = {'role': 'user', 'content': []} prompt, images = prompt prompt_first = True @@ -341,7 +339,7 @@ def _re_format_prompt_images_pair(prompt: tuple) -> dict: # 'image_url': means url or local path to image. # 'image_data': means PIL.Image.Image object. if isinstance(image, str): - image = load_image(image) + image = load_from_url(image, ImageMediaIO(), allowed_media_domains=allowed_media_domains) item = {'type': 'image_data', 'image_data': {'data': image}} elif isinstance(image, PIL.Image.Image): item = {'type': 'image_data', 'image_data': {'data': image}} diff --git a/tests/test_lmdeploy/test_content_merge.py b/tests/test_lmdeploy/test_content_merge.py index 5b50706ed9..05b3f2a845 100644 --- a/tests/test_lmdeploy/test_content_merge.py +++ b/tests/test_lmdeploy/test_content_merge.py @@ -387,6 +387,26 @@ def fake_load_from_url(data_src, media_io, allowed_media_domains=None): ] +def test_format_prompts_passes_allowed_media_domains(monkeypatch): + """Tuple prompt URL loading should honor the configured domain allowlist.""" + image = Image.new('RGB', (1, 1)) + load_calls = [] + + def fake_load_from_url(data_src, media_io, allowed_media_domains=None): + load_calls.append((data_src, type(media_io).__name__, allowed_media_domains)) + return image + + monkeypatch.setattr(multimodal_module, 'load_from_url', fake_load_from_url) + + prompts = MultimodalProcessor.format_prompts(('describe', 'https://example.com/a.png'), + allowed_media_domains=['example.com']) + + assert load_calls == [('https://example.com/a.png', 'ImageMediaIO', ['example.com'])] + assert prompts[0][0]['content'][0] == {'type': 'text', 'text': 'describe'} + assert prompts[0][0]['content'][1]['type'] == 'image_data' + assert prompts[0][0]['content'][1]['image_data']['data'] is image + + def test_async_parse_multimodal_item_preserves_tool_image_content(monkeypatch): """Tool result images should be parsed in place like vLLM.""" load_calls = [] From d0edf51d94fed341b125dbf1008341639a83213f Mon Sep 17 00:00:00 2001 From: zxy Date: Fri, 10 Jul 2026 17:39:50 +0800 Subject: [PATCH 5/5] fix: normalize allowed media domains --- lmdeploy/vl/media/connection.py | 1 + tests/test_lmdeploy/test_vl/test_safe_url.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/lmdeploy/vl/media/connection.py b/lmdeploy/vl/media/connection.py index 2ff1a6e496..f6c5c27691 100644 --- a/lmdeploy/vl/media/connection.py +++ b/lmdeploy/vl/media/connection.py @@ -57,6 +57,7 @@ def _is_safe_url(url: str) -> tuple[bool, str]: def _load_http_url(url_spec: ParseResult, media_io: MediaIO[_M], allowed_media_domains: list[str] | None = None) -> _M: url = url_spec.geturl() + allowed_media_domains = {domain.lower() for domain in allowed_media_domains} if allowed_media_domains else None fetch_timeout = 10 if isinstance(media_io, ImageMediaIO): fetch_timeout = int(os.environ.get('LMDEPLOY_IMAGE_FETCH_TIMEOUT', 10)) diff --git a/tests/test_lmdeploy/test_vl/test_safe_url.py b/tests/test_lmdeploy/test_vl/test_safe_url.py index 828bfc4673..60c3ee1360 100644 --- a/tests/test_lmdeploy/test_vl/test_safe_url.py +++ b/tests/test_lmdeploy/test_vl/test_safe_url.py @@ -59,6 +59,17 @@ def test_load_http_url_allowed_media_domains_exact_match(mock_safe, mock_get): allowed_media_domains=['example.com']) == 'loaded' +@patch('requests.Session.get') +@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +def test_load_http_url_allowed_media_domains_normalizes_case(mock_safe, mock_get): + media_io = MagicMock() + media_io.load_bytes.return_value = 'loaded' + mock_get.return_value = MagicMock(content=b'data', status_code=200, is_redirect=False) + + assert _load_http_url(urlparse('https://example.com/img.jpg'), media_io, + allowed_media_domains=['Example.COM']) == 'loaded' + + @patch('requests.Session.get') @patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) def test_load_http_url_allowed_media_domains_rejects_subdomain(mock_safe, mock_get):