fix(python-backends): re-attach media markers under use_tokenizer_template (#11621)

backend/python/common/python_utils.py

@@ -37,6 +37,46 @@ def parse_options(options_list):
     return opts
 
 
+def attach_media_parts(messages_dicts, n_images=0, n_videos=0):
+    """Rebuild the last user message as content *parts* carrying media markers.
+
+    Backends that let the tokenizer do the templating hand plain string content
+    to ``apply_chat_template``, but a chat template only emits the model's own
+    media tokens (``<|vision_start|><|image_pad|><|vision_end|>`` for the
+    Qwen-VL family, and the equivalents elsewhere) when the content is a list
+    of parts. Without those markers the engine's multimodal processor finds
+    nothing to substitute and silently discards the pixels, even though they
+    were forwarded correctly out of band.
+
+    Returns a new list whose last user message has
+    ``[{"type": "image"} * n_images, {"type": "video"} * n_videos, text]`` as
+    its content, or ``None`` when there is nothing to attach - no media, no
+    user turn, or content that is already a list of parts - so the caller can
+    keep using the original string-content list.
+    """
+    if not n_images and not n_videos:
+        return None
+    idx = next(
+        (
+            i
+            for i in reversed(range(len(messages_dicts)))
+            if messages_dicts[i].get("role") == "user"
+        ),
+        None,
+    )
+    if idx is None:
+        return None
+    text = messages_dicts[idx].get("content") or ""
+    if not isinstance(text, str):
+        return None
+    parts = [{"type": "image"}] * n_images + [{"type": "video"}] * n_videos
+    if text:
+        parts.append({"type": "text", "text": text})
+    patched = list(messages_dicts)
+    patched[idx] = dict(patched[idx], content=parts)
+    return patched
+
+
 def messages_to_dicts(proto_messages):
     """Convert proto ``Message`` objects to dicts suitable for ``apply_chat_template``.
 

backend/python/common/python_utils_test.py

@@ -14,7 +14,7 @@
 import types
 import unittest
 
-from python_utils import messages_to_dicts, parse_options
+from python_utils import attach_media_parts, messages_to_dicts, parse_options
 
 
 def _msg(**fields):
@@ -118,5 +118,63 @@ def test_tool_calls_invalid_json_dropped(self):
         self.assertNotIn("tool_calls", out[0])
 
 
+class TestAttachMediaParts(unittest.TestCase):
+    def test_image_marker_added_to_last_user_turn(self):
+        messages = [
+            {"role": "system", "content": "be brief"},
+            {"role": "user", "content": "first"},
+            {"role": "assistant", "content": "ok"},
+            {"role": "user", "content": "how high is the water?"},
+        ]
+        out = attach_media_parts(messages, n_images=1)
+        self.assertEqual(
+            out[3]["content"],
+            [{"type": "image"}, {"type": "text", "text": "how high is the water?"}],
+        )
+        # Earlier turns and the input list itself are untouched.
+        self.assertEqual(out[:3], messages[:3])
+        self.assertEqual(messages[3]["content"], "how high is the water?")
+
+    def test_counts_and_order_images_then_videos(self):
+        out = attach_media_parts(
+            [{"role": "user", "content": "describe"}], n_images=2, n_videos=1
+        )
+        self.assertEqual(
+            out[0]["content"],
+            [
+                {"type": "image"},
+                {"type": "image"},
+                {"type": "video"},
+                {"type": "text", "text": "describe"},
+            ],
+        )
+
+    def test_empty_text_yields_media_only_parts(self):
+        out = attach_media_parts([{"role": "user", "content": ""}], n_images=1)
+        self.assertEqual(out[0]["content"], [{"type": "image"}])
+
+    def test_other_message_keys_are_preserved(self):
+        out = attach_media_parts(
+            [{"role": "user", "content": "hi", "name": "bob"}], n_images=1
+        )
+        self.assertEqual(out[0]["name"], "bob")
+
+    def test_no_media_is_a_no_op(self):
+        self.assertIsNone(attach_media_parts([{"role": "user", "content": "hi"}]))
+
+    def test_no_user_turn_is_a_no_op(self):
+        self.assertIsNone(
+            attach_media_parts([{"role": "system", "content": "hi"}], n_images=1)
+        )
+
+    def test_content_already_parts_is_a_no_op(self):
+        self.assertIsNone(
+            attach_media_parts(
+                [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],
+                n_images=1,
+            )
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

backend/python/sglang/backend.py

@@ -40,6 +40,7 @@
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
+from python_utils import attach_media_parts
 from grpc_auth import get_auth_interceptors
 
 # sglang imports. Engine is the stable public entry point; parser modules
@@ -355,6 +356,24 @@ def _build_prompt(self, request) -> str:
         if request.Metadata.get("enable_thinking", "").lower() == "true":
             template_kwargs["enable_thinking"] = True
 
+        # sglang locates the attached images/videos by scanning the rendered
+        # prompt for the model's own media token, so the template has to be
+        # given content *parts* - string content renders a prompt with no
+        # placeholder and the media are dropped without a word (#11621).
+        media_dicts = attach_media_parts(
+            messages_dicts, len(request.Images), len(request.Videos)
+        )
+        if media_dicts is not None:
+            try:
+                return self.tokenizer.apply_chat_template(media_dicts, **template_kwargs)
+            except Exception as e:
+                # A text-only template cannot iterate content parts; fall
+                # through to the text-only prompt instead of failing.
+                print(
+                    f"chat template rejected multimodal content parts: {e!r}",
+                    file=sys.stderr,
+                )
+
         try:
             return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
         except TypeError:

backend/python/vllm/backend.py

@@ -20,6 +20,7 @@
 import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
+from python_utils import attach_media_parts
 from grpc_auth import get_auth_interceptors
 
 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -576,13 +577,33 @@ async def _predict(self, request, context, streaming=False):
             if request.Metadata.get("enable_thinking", "").lower() == "true":
                 template_kwargs["enable_thinking"] = True
 
-            try:
-                prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
-            except TypeError:
-                # Some tokenizers don't support tools/enable_thinking kwargs — retry without them
-                prompt = self.tokenizer.apply_chat_template(
-                    messages_dicts, tokenize=False, add_generation_prompt=True
-                )
+            # vLLM substitutes multi_modal_data into the model's own media
+            # token, so the template has to be given content *parts* - string
+            # content renders a prompt with no placeholder and the media are
+            # dropped without a word (#11621).
+            prompt = None
+            media_dicts = attach_media_parts(
+                messages_dicts, len(image_data), len(video_data)
+            )
+            if media_dicts is not None:
+                try:
+                    prompt = self.tokenizer.apply_chat_template(media_dicts, **template_kwargs)
+                except Exception as e:
+                    # A text-only template cannot iterate content parts; fall
+                    # through to the text-only prompt instead of failing.
+                    print(
+                        f"chat template rejected multimodal content parts: {e!r}",
+                        file=sys.stderr,
+                    )
+
+            if prompt is None:
+                try:
+                    prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
+                except TypeError:
+                    # Some tokenizers don't support tools/enable_thinking kwargs — retry without them
+                    prompt = self.tokenizer.apply_chat_template(
+                        messages_dicts, tokenize=False, add_generation_prompt=True
+                    )
 
         # Generate text using the LLM engine
         request_id = random_uuid()