fix: restrict remote media domains

docs/en/multi_modal/multimodal_inputs.md

@@ -393,6 +393,12 @@ ______________________________________________________________________
 
 ## Local Files and Base64
 
+When the API server is exposed to untrusted clients, use `--allowed-media-domains` at startup to restrict which hostnames HTTP(S) media URLs can fetch from and reduce SSRF risk. The option matches exact hostnames; for example, `--allowed-media-domains example.com` allows `https://example.com/...` but not `https://cdn.example.com/...`.
+
+```shell
+lmdeploy serve api_server <model_path> --allowed-media-domains example.com
+```
+
 In addition to HTTP URLs, lmdeploy accepts:
 
 - **Local file paths** via `file://` scheme: `file:///absolute/path/to/file.jpg`

docs/zh_cn/multi_modal/multimodal_inputs.md

@@ -392,6 +392,12 @@ ______________________________________________________________________
 
 ## 本地文件与 Base64
 
+当 API 服务面向不可信客户端时，建议在启动服务时使用 `--allowed-media-domains` 限制 HTTP(S) 媒体 URL 可访问的域名，降低 SSRF 风险。该参数按精确 hostname 匹配；例如 `--allowed-media-domains example.com` 只允许 `https://example.com/...`，不允许 `https://cdn.example.com/...`。
+
+```shell
+lmdeploy serve api_server <model_path> --allowed-media-domains example.com
+```
+
 除 HTTP URL 外，lmdeploy 还支持：
 
 - **本地文件路径**，使用 `file://` 协议：`file:///absolute/path/to/file.jpg`

lmdeploy/api.py

@@ -19,6 +19,7 @@ def pipeline(model_path: str,
              max_log_len: int | None = None,
              trust_remote_code: bool = False,
              speculative_config: SpeculativeConfig | None = None,
+             allowed_media_domains: list[str] | None = None,
              **kwargs):
     """Create a pipeline for inference.
 
@@ -43,6 +44,7 @@ def pipeline(model_path: str,
             being printed in log.
         trust_remote_code: whether to trust remote code from model repositories.
         speculative_config: speculative decoding configuration.
+        allowed_media_domains: Optional HTTP(S) media URL domain allowlist.
         **kwargs: additional keyword arguments passed to the pipeline.
 
     Returns:
@@ -76,6 +78,7 @@ def pipeline(model_path: str,
                     max_log_len=max_log_len,
                     trust_remote_code=trust_remote_code,
                     speculative_config=speculative_config,
+                    allowed_media_domains=allowed_media_domains,
                     **kwargs)
 
 

lmdeploy/cli/serve.py

@@ -75,6 +75,11 @@ def add_parser_api_server():
                             'engine’s tasks once the maximum number of concurrent requests is '
                             'reached, regardless of any additional requests sent by clients '
                             'concurrently during that time. Default to None.')
+        parser.add_argument('--allowed-media-domains',
+                            nargs='+',
+                            type=str,
+                            default=None,
+                            help='Exact hostnames allowed for HTTP(S) media URLs.')
         # common args
         ArgumentHelper.backend(parser)
         ArgumentHelper.log_level(parser)
@@ -323,6 +328,7 @@ def api_server(args):
                 reasoning_parser=args.reasoning_parser,
                 tool_call_parser=args.tool_call_parser,
                 speculative_config=speculative_config,
+                allowed_media_domains=args.allowed_media_domains,
                 generation_config=args.generation_config,
             )
         else:
@@ -356,6 +362,7 @@ def api_server(args):
                 reasoning_parser=args.reasoning_parser,
                 tool_call_parser=args.tool_call_parser,
                 speculative_config=speculative_config,
+                allowed_media_domains=args.allowed_media_domains,
                 generation_config=args.generation_config,
             )
 

lmdeploy/pipeline.py

@@ -40,6 +40,7 @@ def __init__(self,
                  max_log_len: int | None = None,
                  trust_remote_code: bool = False,
                  speculative_config: SpeculativeConfig | None = None,
+                 allowed_media_domains: list[str] | None = None,
                  **kwargs):
         """Initialize Pipeline.
 
@@ -51,6 +52,7 @@ def __init__(self,
             max_log_len: Max number of prompt characters or prompt tokens being printed in log.
             trust_remote_code: whether to trust remote code from model repositories.
             speculative_config: Speculative decoding configuration.
+            allowed_media_domains: Optional HTTP(S) media URL domain allowlist.
             **kwargs: Additional keyword arguments.
         """
 
@@ -82,11 +84,13 @@ def __init__(self,
                                            max_log_len=max_log_len,
                                            trust_remote_code=trust_remote_code,
                                            speculative_config=speculative_config,
+                                           allowed_media_domains=allowed_media_domains,
                                            **kwargs)
         self.internal_thread = _EventLoopThread(daemon=True)
         self.limiter: asyncio.Semaphore = None
         self.session_mgr = self.async_engine.session_mgr
         self.backend_config = self.async_engine.backend_config
+        self.allowed_media_domains = allowed_media_domains
         self.async_engine.start_loop(self.internal_thread.loop, use_async_api=False)
 
     def infer(self,
@@ -112,7 +116,7 @@ def infer(self,
         """
         is_single = self._is_single(prompts)
         # format prompts to openai message format, which is a list of dicts
-        prompts = MultimodalProcessor.format_prompts(prompts)
+        prompts = MultimodalProcessor.format_prompts(prompts, allowed_media_domains=self.allowed_media_domains)
         pbar = tqdm.tqdm(total=len(prompts)) if use_tqdm else None
         outputs = []
         try:
@@ -162,7 +166,7 @@ def stream_infer(self,
         Returns:
             Iterator: A generator that yields the output (i.e. instance of class ``Response``) of the inference.
         """
-        prompts = MultimodalProcessor.format_prompts(prompts)
+        prompts = MultimodalProcessor.format_prompts(prompts, allowed_media_domains=self.allowed_media_domains)
         requests = self._request_generator(prompts,
                                            sessions=sessions,
                                            gen_config=gen_config,
@@ -201,7 +205,7 @@ def chat(self,
             session = self.session_mgr.get()
         session.update(prompt=prompt, response=None)
 
-        prompt = MultimodalProcessor.format_prompts(prompt)
+        prompt = MultimodalProcessor.format_prompts(prompt, allowed_media_domains=self.allowed_media_domains)
 
         sequence_start = session.step == 0
         generator = self.stream_infer(prompts=prompt,

lmdeploy/serve/core/async_engine.py

@@ -114,6 +114,7 @@ def __init__(self,
                  max_log_len: int | None = None,
                  trust_remote_code: bool = False,
                  speculative_config: SpeculativeConfig | None = None,
+                 allowed_media_domains: list[str] | None = None,
                  **kwargs) -> None:
         logger.info(f'input backend={backend}, backend_config={backend_config}')
         logger.info(f'speculative_config={speculative_config}')
@@ -122,7 +123,9 @@ def __init__(self,
         self.model_name = model_name if model_name else model_path
         self.chat_template = get_chat_template(model_path, chat_template_config, trust_remote_code=trust_remote_code)
         self.tokenizer = Tokenizer(model_path, trust_remote_code=trust_remote_code)
-        self.prompt_processor = MultimodalProcessor(self.tokenizer, self.chat_template)
+        self.prompt_processor = MultimodalProcessor(self.tokenizer,
+                                                    self.chat_template,
+                                                    allowed_media_domains=allowed_media_domains)
         self.hf_gen_cfg = get_hf_gen_cfg(model_path, trust_remote_code=trust_remote_code)
         self.arch, self.hf_cfg = get_model_arch(model_path, trust_remote_code=trust_remote_code)
         self.session_len = (_get_and_verify_max_len(self.hf_cfg, None)

lmdeploy/serve/core/vl_async_engine.py

@@ -18,6 +18,7 @@ def __init__(self,
                  backend_config: TurbomindEngineConfig | PytorchEngineConfig | None = None,
                  vision_config: VisionConfig | None = None,
                  trust_remote_code: bool = False,
+                 allowed_media_domains: list[str] | None = None,
                  **kwargs) -> None:
         from lmdeploy.serve.processors import MultimodalProcessor
         from lmdeploy.utils import try_import_deeplink
@@ -40,12 +41,14 @@ def __init__(self,
                          backend=backend,
                          backend_config=backend_config,
                          trust_remote_code=trust_remote_code,
+                         allowed_media_domains=allowed_media_domains,
                          **kwargs)
         # Update prompt_processor to support multimodal processing
         self.prompt_processor = MultimodalProcessor(self.tokenizer,
                                                     self.chat_template,
                                                     vl_encoder=self.vl_encoder,
-                                                    backend=backend)
+                                                    backend=backend,
+                                                    allowed_media_domains=allowed_media_domains)
         if self.model_name == 'base':
             raise RuntimeError(
                 'please specify chat template as guided in https://lmdeploy.readthedocs.io/en/latest/inference/vl_pipeline.html#set-chat-template'  # noqa: E501

lmdeploy/serve/openai/api_server.py

@@ -1526,6 +1526,7 @@ def serve(model_path: str,
           allow_terminate_by_client: bool = False,
           enable_abort_handling: bool = False,
           speculative_config: SpeculativeConfig | None = None,
+          allowed_media_domains: list[str] | None = None,
           generation_config: str = 'auto',
           **kwargs):
     """An example to perform model inference through the command line
@@ -1578,6 +1579,8 @@ def serve(model_path: str,
         reasoning_parser (str): The reasoning parser name.
         tool_call_parser (str): The tool call parser name.
         allow_terminate_by_client (bool): Allow request from client to terminate server.
+        allowed_media_domains (list[str] | None): Optional exact hostname
+            allowlist for HTTP(S) media URLs.
     """
     if os.getenv('TM_LOG_LEVEL') is None:
         os.environ['TM_LOG_LEVEL'] = log_level
@@ -1618,6 +1621,7 @@ def serve(model_path: str,
                                                     max_log_len=max_log_len,
                                                     trust_remote_code=trust_remote_code,
                                                     speculative_config=speculative_config,
+                                                    allowed_media_domains=allowed_media_domains,
                                                     **kwargs)
     set_parsers(reasoning_parser, tool_call_parser)
 

lmdeploy/serve/processors/multimodal.py

@@ -37,19 +37,22 @@ def __init__(self,
                  tokenizer: Tokenizer,
                  chat_template: BaseChatTemplate,
                  vl_encoder=None,
-                 backend: str | None = None):
+                 backend: str | None = None,
+                 allowed_media_domains: list[str] | None = None):
         """Initialize MultimodalProcessor.
 
         Args:
             tokenizer: Tokenizer instance for encoding prompts.
             chat_template: Chat template instance for message processing.
             vl_encoder: Optional ImageEncoder instance for multimodal processing.
             backend: Optional backend name ('turbomind' or 'pytorch') for multimodal processing.
+            allowed_media_domains: Optional HTTP(S) media URL domain allowlist.
         """
         self.tokenizer = tokenizer
         self.chat_template = chat_template
         self.vl_encoder = vl_encoder
         self.backend = backend
+        self.allowed_media_domains = allowed_media_domains
 
     @staticmethod
     def merge_message_content(msg: dict) -> dict:
@@ -103,8 +106,11 @@ def merge_message_content(msg: dict) -> dict:
         return result
 
     @staticmethod
-    def _parse_multimodal_item(i: int, in_messages: list[dict], out_messages: list[dict], media_io_kwargs: dict[str,
-                                                                                                                Any]):
+    def _parse_multimodal_item(i: int,
+                               in_messages: list[dict],
+                               out_messages: list[dict],
+                               media_io_kwargs: dict[str, Any],
+                               allowed_media_domains: list[str] | None = None):
         """Synchronous helper to parse a single multimodal message item."""
         role = in_messages[i]['role']
         content = in_messages[i]['content']
@@ -153,21 +159,29 @@ def _require_data_src():
                 if isinstance(data_src, PIL.Image.Image):
                     data = data_src
                 elif isinstance(data_src, str):
-                    data = load_from_url(data_src, ImageMediaIO(**media_io_kwargs.get('image', {})))
+                    data = load_from_url(data_src,
+                                         ImageMediaIO(**media_io_kwargs.get('image', {})),
+                                         allowed_media_domains=allowed_media_domains)
                 else:
                     raise ValueError(f'Invalid multimodal image item at index {i}: {item}. '
                                      'Expected a str URL/path/data URL or PIL.Image.Image.')
             elif item_type in ('video_url', 'video'):
                 modality = Modality.VIDEO
                 data, metadata = load_from_url(
-                    _require_data_src(), VideoMediaIO(image_io=ImageMediaIO(), **media_io_kwargs.get('video', {})))
+                    _require_data_src(),
+                    VideoMediaIO(image_io=ImageMediaIO(), **media_io_kwargs.get('video', {})),
+                    allowed_media_domains=allowed_media_domains)
                 item_params['video_metadata'] = metadata
             elif item_type in ('audio_url', 'audio'):
                 modality = Modality.AUDIO
-                data = load_from_url(_require_data_src(), AudioMediaIO(**media_io_kwargs.get('audio', {})))
+                data = load_from_url(_require_data_src(),
+                                     AudioMediaIO(**media_io_kwargs.get('audio', {})),
+                                     allowed_media_domains=allowed_media_domains)
             elif item_type in ('time_series_url', 'time_series'):
                 modality = Modality.TIME_SERIES
-                data = load_from_url(_require_data_src(), TimeSeriesMediaIO(**media_io_kwargs.get('time_series', {})))
+                data = load_from_url(_require_data_src(),
+                                     TimeSeriesMediaIO(**media_io_kwargs.get('time_series', {})),
+                                     allowed_media_domains=allowed_media_domains)
             else:
                 raise NotImplementedError(f'unknown type: {item_type}')
 
@@ -177,7 +191,8 @@ def _require_data_src():
 
     @staticmethod
     async def async_parse_multimodal_item(messages: list[dict],
-                                          media_io_kwargs: dict[str, Any] | None = None) -> list[dict]:
+                                          media_io_kwargs: dict[str, Any] | None = None,
+                                          allowed_media_domains: list[str] | None = None) -> list[dict]:
         """Convert user-input multimodal data into GPT4V message format."""
         if isinstance(messages, dict):
             messages = [messages]
@@ -189,7 +204,7 @@ async def async_parse_multimodal_item(messages: list[dict],
 
         await asyncio.gather(*[
             loop.run_in_executor(None, MultimodalProcessor._parse_multimodal_item, i, messages, out_messages,
-                                 media_io_kwargs) for i in range(len(messages))
+                                 media_io_kwargs, allowed_media_domains) for i in range(len(messages))
         ])
         return out_messages
 
@@ -266,7 +281,7 @@ async def get_prompt_input(self,
             raise RuntimeError(f'unsupported prompt type: {type(prompt)}')
 
     @staticmethod
-    def format_prompts(prompts: Any) -> list[dict]:
+    def format_prompts(prompts: Any, allowed_media_domains: list[str] | None = None) -> list[dict]:
         """Format prompts."""
         if not isinstance(prompts, list):
             prompts = [prompts]
@@ -279,7 +294,8 @@ def format_prompts(prompts: Any) -> list[dict]:
         if all(MultimodalProcessor._is_str_images_pair(prompt) for prompt in prompts):
             # batch of (prompt, image or [images]) or (image or [images], prompt) ->
             # [[openai_gpt4v_message], [openai_gpt4v_message], ...]
-            return [[MultimodalProcessor._re_format_prompt_images_pair(prompt)] for prompt in prompts]
+            return [[MultimodalProcessor._re_format_prompt_images_pair(prompt, allowed_media_domains)]
+                    for prompt in prompts]
         raise ValueError(f'Unsupported prompts: {prompts}. Only support str, openai message format, '
                          'or (prompt, image or [images]) or (image or [images], prompt) pair.')
 
@@ -310,10 +326,8 @@ def _is_image_list(obj) -> bool:
         return isinstance(obj, list) and all(MultimodalProcessor._is_image(img) for img in obj)
 
     @staticmethod
-    def _re_format_prompt_images_pair(prompt: tuple) -> dict:
+    def _re_format_prompt_images_pair(prompt: tuple, allowed_media_domains: list[str] | None = None) -> dict:
         """Reformat the prompt to openai message format."""
-        from lmdeploy.vl import load_image
-
         messages = {'role': 'user', 'content': []}
         prompt, images = prompt
         prompt_first = True
@@ -326,7 +340,7 @@ def _re_format_prompt_images_pair(prompt: tuple) -> dict:
             # 'image_url': means url or local path to image.
             # 'image_data': means PIL.Image.Image object.
             if isinstance(image, str):
-                image = load_image(image)
+                image = load_from_url(image, ImageMediaIO(), allowed_media_domains=allowed_media_domains)
                 item = {'type': 'image_data', 'image_data': {'data': image}}
             elif isinstance(image, PIL.Image.Image):
                 item = {'type': 'image_data', 'image_data': {'data': image}}
@@ -400,7 +414,9 @@ async def _get_multimodal_prompt_input(self,
         """Process multimodal prompt and return processed data for inference
         engines."""
         chat_template = self.chat_template if do_preprocess else BaseChatTemplate()
-        messages = await self.async_parse_multimodal_item(messages, media_io_kwargs)
+        messages = await self.async_parse_multimodal_item(messages,
+                                                          media_io_kwargs,
+                                                          allowed_media_domains=self.allowed_media_domains)
 
         if self.backend == 'turbomind':
             if self.vl_encoder._uses_new_preprocess: