diff --git a/backend/python/common/python_utils.py b/backend/python/common/python_utils.py index c89813e2c5da..88ec0a530cc6 100644 --- a/backend/python/common/python_utils.py +++ b/backend/python/common/python_utils.py @@ -37,6 +37,46 @@ def parse_options(options_list): return opts +def attach_media_parts(messages_dicts, n_images=0, n_videos=0): + """Rebuild the last user message as content *parts* carrying media markers. + + Backends that let the tokenizer do the templating hand plain string content + to ``apply_chat_template``, but a chat template only emits the model's own + media tokens (``<|vision_start|><|image_pad|><|vision_end|>`` for the + Qwen-VL family, and the equivalents elsewhere) when the content is a list + of parts. Without those markers the engine's multimodal processor finds + nothing to substitute and silently discards the pixels, even though they + were forwarded correctly out of band. + + Returns a new list whose last user message has + ``[{"type": "image"} * n_images, {"type": "video"} * n_videos, text]`` as + its content, or ``None`` when there is nothing to attach - no media, no + user turn, or content that is already a list of parts - so the caller can + keep using the original string-content list. + """ + if not n_images and not n_videos: + return None + idx = next( + ( + i + for i in reversed(range(len(messages_dicts))) + if messages_dicts[i].get("role") == "user" + ), + None, + ) + if idx is None: + return None + text = messages_dicts[idx].get("content") or "" + if not isinstance(text, str): + return None + parts = [{"type": "image"}] * n_images + [{"type": "video"}] * n_videos + if text: + parts.append({"type": "text", "text": text}) + patched = list(messages_dicts) + patched[idx] = dict(patched[idx], content=parts) + return patched + + def messages_to_dicts(proto_messages): """Convert proto ``Message`` objects to dicts suitable for ``apply_chat_template``. diff --git a/backend/python/common/python_utils_test.py b/backend/python/common/python_utils_test.py index c395ce92d0a5..d12bac9c5392 100644 --- a/backend/python/common/python_utils_test.py +++ b/backend/python/common/python_utils_test.py @@ -14,7 +14,7 @@ import types import unittest -from python_utils import messages_to_dicts, parse_options +from python_utils import attach_media_parts, messages_to_dicts, parse_options def _msg(**fields): @@ -118,5 +118,63 @@ def test_tool_calls_invalid_json_dropped(self): self.assertNotIn("tool_calls", out[0]) +class TestAttachMediaParts(unittest.TestCase): + def test_image_marker_added_to_last_user_turn(self): + messages = [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "how high is the water?"}, + ] + out = attach_media_parts(messages, n_images=1) + self.assertEqual( + out[3]["content"], + [{"type": "image"}, {"type": "text", "text": "how high is the water?"}], + ) + # Earlier turns and the input list itself are untouched. + self.assertEqual(out[:3], messages[:3]) + self.assertEqual(messages[3]["content"], "how high is the water?") + + def test_counts_and_order_images_then_videos(self): + out = attach_media_parts( + [{"role": "user", "content": "describe"}], n_images=2, n_videos=1 + ) + self.assertEqual( + out[0]["content"], + [ + {"type": "image"}, + {"type": "image"}, + {"type": "video"}, + {"type": "text", "text": "describe"}, + ], + ) + + def test_empty_text_yields_media_only_parts(self): + out = attach_media_parts([{"role": "user", "content": ""}], n_images=1) + self.assertEqual(out[0]["content"], [{"type": "image"}]) + + def test_other_message_keys_are_preserved(self): + out = attach_media_parts( + [{"role": "user", "content": "hi", "name": "bob"}], n_images=1 + ) + self.assertEqual(out[0]["name"], "bob") + + def test_no_media_is_a_no_op(self): + self.assertIsNone(attach_media_parts([{"role": "user", "content": "hi"}])) + + def test_no_user_turn_is_a_no_op(self): + self.assertIsNone( + attach_media_parts([{"role": "system", "content": "hi"}], n_images=1) + ) + + def test_content_already_parts_is_a_no_op(self): + self.assertIsNone( + attach_media_parts( + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + n_images=1, + ) + ) + + if __name__ == "__main__": unittest.main() diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index 0d38c6b7d2c9..5db4aa9679c8 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -40,6 +40,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) +from python_utils import attach_media_parts from grpc_auth import get_auth_interceptors # sglang imports. Engine is the stable public entry point; parser modules @@ -355,6 +356,24 @@ def _build_prompt(self, request) -> str: if request.Metadata.get("enable_thinking", "").lower() == "true": template_kwargs["enable_thinking"] = True + # sglang locates the attached images/videos by scanning the rendered + # prompt for the model's own media token, so the template has to be + # given content *parts* - string content renders a prompt with no + # placeholder and the media are dropped without a word (#11621). + media_dicts = attach_media_parts( + messages_dicts, len(request.Images), len(request.Videos) + ) + if media_dicts is not None: + try: + return self.tokenizer.apply_chat_template(media_dicts, **template_kwargs) + except Exception as e: + # A text-only template cannot iterate content parts; fall + # through to the text-only prompt instead of failing. + print( + f"chat template rejected multimodal content parts: {e!r}", + file=sys.stderr, + ) + try: return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) except TypeError: diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 7d85daf944d1..ba17857888a6 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -20,6 +20,7 @@ import grpc sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) +from python_utils import attach_media_parts from grpc_auth import get_auth_interceptors from vllm.engine.arg_utils import AsyncEngineArgs @@ -576,13 +577,33 @@ async def _predict(self, request, context, streaming=False): if request.Metadata.get("enable_thinking", "").lower() == "true": template_kwargs["enable_thinking"] = True - try: - prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) - except TypeError: - # Some tokenizers don't support tools/enable_thinking kwargs — retry without them - prompt = self.tokenizer.apply_chat_template( - messages_dicts, tokenize=False, add_generation_prompt=True - ) + # vLLM substitutes multi_modal_data into the model's own media + # token, so the template has to be given content *parts* - string + # content renders a prompt with no placeholder and the media are + # dropped without a word (#11621). + prompt = None + media_dicts = attach_media_parts( + messages_dicts, len(image_data), len(video_data) + ) + if media_dicts is not None: + try: + prompt = self.tokenizer.apply_chat_template(media_dicts, **template_kwargs) + except Exception as e: + # A text-only template cannot iterate content parts; fall + # through to the text-only prompt instead of failing. + print( + f"chat template rejected multimodal content parts: {e!r}", + file=sys.stderr, + ) + + if prompt is None: + try: + prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) + except TypeError: + # Some tokenizers don't support tools/enable_thinking kwargs — retry without them + prompt = self.tokenizer.apply_chat_template( + messages_dicts, tokenize=False, add_generation_prompt=True + ) # Generate text using the LLM engine request_id = random_uuid()