(Openinference Migration: Langchain): Capture multimodal image content (OpenAI image_url and Anthropic image blocks) as Blob/Uri message parts.

instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added

@@ -0,0 +1 @@
+(Openinference Migration: Langchain) - Capture multimodal image content (OpenAI ``image_url`` and Anthropic ``image`` blocks) as ``Blob``/``Uri`` message parts.

instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py

@@ -40,6 +40,7 @@
     WorkflowInvocation,
 )
 from opentelemetry.util.genai.types import (
+    InputMessage,
     MessagePart,
     OutputMessage,
     Text,
@@ -73,7 +74,7 @@ def on_chain_start(
         operation = classify_chain_run(
             serialized, metadata, kwargs, parent_run_id
         )
-
+        capture_content = self._telemetry_handler.should_capture_content()
         if operation == OperationName.INVOKE_WORKFLOW:
             workflow_name = kwargs.get("name") or serialized.get("name")
             workflow_name_override = (
@@ -82,7 +83,8 @@ def on_chain_start(
             workflow = self._telemetry_handler.workflow(
                 name=workflow_name_override or workflow_name
             )
-            workflow.input_messages = make_input_message(inputs)
+            if capture_content:
+                workflow.input_messages = make_input_message(inputs)
             self._invocation_manager.add_invocation_state(
                 run_id, parent_run_id, workflow
             )
@@ -107,7 +109,8 @@ def on_chain_start(
                     agent = self._telemetry_handler.invoke_local_agent(
                         agent_name=suggested_agent_name,
                     )
-                    agent.input_messages = make_input_message(inputs)
+                    if capture_content:
+                        agent.input_messages = make_input_message(inputs)
 
                     if metadata:
                         agent.agent_id = metadata.get("agent_id")
@@ -162,7 +165,8 @@ def on_chain_end(
             self._invocation_manager.delete_invocation_state(run_id)
             return
 
-        invocation.output_messages = make_last_output_message(outputs)
+        if self._telemetry_handler.should_capture_content():
+            invocation.output_messages = make_last_output_message(outputs)
 
         invocation.stop()
 
@@ -270,7 +274,9 @@ def on_chat_model_start(
         # :func:`to_input_messages` produce spec-conformant ``InputMessage`` s
         # with proper roles, tool-call requests, tool results, and reasoning.
         flattened: list[BaseMessage] = [msg for sub in messages for msg in sub]
-        input_messages = to_input_messages(flattened)
+        input_messages: list[InputMessage] = []
+        if self._telemetry_handler.should_capture_content():
+            input_messages = to_input_messages(flattened)
 
         llm_invocation = self._telemetry_handler.inference(
             provider,

instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py

@@ -19,6 +19,7 @@
     gen_ai_attributes as GenAIAttributes,
 )
 from opentelemetry.util.genai.types import (
+    Blob,
     FunctionToolDefinition,
     InputMessage,
     MessagePart,
@@ -29,6 +30,7 @@
     ToolCallResponse,
     ToolDefinition,
 )
+from opentelemetry.util.genai.utils import decode_base64, image_from_url
 
 # Mapping from LangChain ``ls_provider`` metadata values to the well-known
 # ``gen_ai.provider.name`` values defined by the GenAI semantic conventions.
@@ -75,6 +77,59 @@ def _normalize_role(message: BaseMessage) -> str:
     return _ROLE_MAP.get(message.type, message.type)
 
 
+def _media_part(item: dict[str, Any]) -> MessagePart | None:
+    """Convert a LangChain multimodal image content block into a media part.
+
+    Handles the two shapes LangChain chat models accept:
+
+    - OpenAI style ``{"type": "image_url", "image_url": {"url": ...}}`` (or a
+      bare ``"image_url": "..."`` string). A ``data:<mime>;base64,<payload>``
+      URL becomes a :class:`Blob`; any other URL becomes a :class:`Uri`.
+    - Anthropic style ``{"type": "image", "source": {...}}`` where ``source``
+      is either ``{"type": "base64", "media_type": ..., "data": ...}`` (→
+      :class:`Blob`) or ``{"type": "url", "url": ...}`` (→ :class:`Uri`).
+    """
+    block_type = item.get("type")
+    if block_type == "image_url":
+        image_url = item.get("image_url")
+        url: str | None = None
+        if isinstance(image_url, str):
+            url = image_url
+        elif isinstance(image_url, dict):
+            image_url_dict = cast(dict[str, Any], image_url)
+            raw_url = image_url_dict.get("url")
+            url = raw_url if isinstance(raw_url, str) else None
+        if not url:
+            return None
+        return image_from_url(url)
+    if block_type == "image":
+        source = item.get("source")
+        if not isinstance(source, dict):
+            return None
+        source_dict = cast(dict[str, Any], source)
+        source_type = source_dict.get("type")
+        if source_type == "base64":
+            data = source_dict.get("data")
+            if not isinstance(data, str):
+                return None
+            decoded = decode_base64(data)
+            if decoded is None:
+                return None
+            media_type = source_dict.get("media_type")
+            return Blob(
+                mime_type=(
+                    media_type if isinstance(media_type, str) else None
+                ),
+                modality="image",
+                content=decoded,
+            )
+        if source_type == "url":
+            source_url = source_dict.get("url")
+            if isinstance(source_url, str) and source_url:
+                return image_from_url(source_url)
+    return None
+
+
 def _content_to_parts(
     content: str | list[str | dict[str, Any]],
 ) -> list[MessagePart]:
@@ -109,6 +164,10 @@ def _content_to_parts(
             )
             if isinstance(reasoning_value, str) and reasoning_value:
                 parts.append(Reasoning(content=reasoning_value))
+        elif block_type in ("image_url", "image"):
+            media = _media_part(item)
+            if media is not None:
+                parts.append(media)
     return parts
 
 
@@ -187,7 +246,11 @@ def _message_parts(message: BaseMessage) -> list[MessagePart]:
 def to_input_messages(
     messages: Iterable[Any],
 ) -> list[InputMessage]:
-    """Convert LangChain messages into spec-conformant ``InputMessage`` s."""
+    """Convert LangChain messages into spec-conformant ``InputMessage`` s.
+
+    Called only when content capture is enabled
+    (``TelemetryHandler.should_capture_content()``).
+    """
     try:
         normalized_messages: Iterable[BaseMessage] = convert_to_messages(
             list(messages)
@@ -216,6 +279,9 @@ def to_output_messages(
     as ``gen_ai.output.messages``. Tool execution results belong on the
     *input* side of the next inference call, not the output side of the
     previous one.
+
+    Called only when content capture is enabled
+    (``TelemetryHandler.should_capture_content()``).
     """
     result: list[OutputMessage] = []
     for message in messages:
@@ -295,6 +361,9 @@ def make_input_message(data: Any) -> list[InputMessage]:
     When no ``messages`` key exists (common in LangGraph state dicts), the
     remaining state fields are serialized as JSON and emitted as a single
     user-role :class:`Text` part.
+
+    Called only when content capture is enabled
+    (``TelemetryHandler.should_capture_content()``).
     """
     if not isinstance(data, dict):
         return []
@@ -329,6 +398,9 @@ def make_output_message(data: Any) -> list[OutputMessage]:
     empty: the underlying per-LLM-call finish reasons are recorded on child
     inference spans, and util-genai filters empty values out of
     ``gen_ai.response.finish_reasons``.
+
+    Called only when content capture is enabled
+    (``TelemetryHandler.should_capture_content()``).
     """
     if not isinstance(data, dict):
         return []
@@ -349,6 +421,9 @@ def make_last_output_message(data: Any) -> list[OutputMessage]:
     For Workflow and AgentInvocation spans, the final AI message best represents
     the actual output. Intermediate AI messages (e.g., tool-call decisions) are
     already captured in child LLM invocation spans.
+
+    Called only when content capture is enabled
+    (``TelemetryHandler.should_capture_content()``).
     """
     all_messages = make_output_message(data)
     if all_messages:

instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml

@@ -0,0 +1,48 @@
+# TODO: this is generated by AI, re-record
+# against the live Anthropic API once an ANTHROPIC_API_KEY is available.
+interactions:
+- request:
+    body: |-
+      {"model": "claude-sonnet-4-5", "max_tokens": 1024, "messages": [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg=="}}]}], "temperature": 0.1}
+    headers:
+      Content-Type:
+      - application/json
+      User-Agent:
+      - !!binary |
+        QW50aHJvcGljL1B5dGhvbiAxLjAuMA==
+      x-api-key:
+      - test_key
+      anthropic-version:
+      - '2023-06-01'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: |-
+        {
+          "id": "msg_01MultimodalImagePlaceholder",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-sonnet-4-5",
+          "content": [
+            {
+              "type": "text",
+              "text": "This is a tiny 1x1 pixel PNG image."
+            }
+          ],
+          "stop_reason": "end_turn",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 16,
+            "output_tokens": 12
+          }
+        }
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 04 Sep 2025 20:00:58 GMT
+    status:
+      code: 200
+      message: OK
+version: 1