diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added new file mode 100644 index 000000000..2c4b09dc1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added @@ -0,0 +1 @@ +(Openinference Migration: Langchain) - Capture multimodal image content (OpenAI ``image_url`` and Anthropic ``image`` blocks) as ``Blob``/``Uri`` message parts. diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index c41f3a386..68e2ca598 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -40,6 +40,7 @@ WorkflowInvocation, ) from opentelemetry.util.genai.types import ( + InputMessage, MessagePart, OutputMessage, Text, @@ -73,7 +74,7 @@ def on_chain_start( operation = classify_chain_run( serialized, metadata, kwargs, parent_run_id ) - + capture_content = self._telemetry_handler.should_capture_content() if operation == OperationName.INVOKE_WORKFLOW: workflow_name = kwargs.get("name") or serialized.get("name") workflow_name_override = ( @@ -82,7 +83,8 @@ def on_chain_start( workflow = self._telemetry_handler.workflow( name=workflow_name_override or workflow_name ) - workflow.input_messages = make_input_message(inputs) + if capture_content: + workflow.input_messages = make_input_message(inputs) self._invocation_manager.add_invocation_state( run_id, parent_run_id, workflow ) @@ -107,7 +109,8 @@ def on_chain_start( agent = self._telemetry_handler.invoke_local_agent( agent_name=suggested_agent_name, ) - agent.input_messages = make_input_message(inputs) + if capture_content: + agent.input_messages = make_input_message(inputs) if metadata: agent.agent_id = metadata.get("agent_id") @@ -162,7 +165,8 @@ def on_chain_end( self._invocation_manager.delete_invocation_state(run_id) return - invocation.output_messages = make_last_output_message(outputs) + if self._telemetry_handler.should_capture_content(): + invocation.output_messages = make_last_output_message(outputs) invocation.stop() @@ -270,7 +274,9 @@ def on_chat_model_start( # :func:`to_input_messages` produce spec-conformant ``InputMessage`` s # with proper roles, tool-call requests, tool results, and reasoning. flattened: list[BaseMessage] = [msg for sub in messages for msg in sub] - input_messages = to_input_messages(flattened) + input_messages: list[InputMessage] = [] + if self._telemetry_handler.should_capture_content(): + input_messages = to_input_messages(flattened) llm_invocation = self._telemetry_handler.inference( provider, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index ef726f1ac..5a9ceaf6e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -19,6 +19,7 @@ gen_ai_attributes as GenAIAttributes, ) from opentelemetry.util.genai.types import ( + Blob, FunctionToolDefinition, InputMessage, MessagePart, @@ -29,6 +30,7 @@ ToolCallResponse, ToolDefinition, ) +from opentelemetry.util.genai.utils import decode_base64, image_from_url # Mapping from LangChain ``ls_provider`` metadata values to the well-known # ``gen_ai.provider.name`` values defined by the GenAI semantic conventions. @@ -75,6 +77,59 @@ def _normalize_role(message: BaseMessage) -> str: return _ROLE_MAP.get(message.type, message.type) +def _media_part(item: dict[str, Any]) -> MessagePart | None: + """Convert a LangChain multimodal image content block into a media part. + + Handles the two shapes LangChain chat models accept: + + - OpenAI style ``{"type": "image_url", "image_url": {"url": ...}}`` (or a + bare ``"image_url": "..."`` string). A ``data:;base64,`` + URL becomes a :class:`Blob`; any other URL becomes a :class:`Uri`. + - Anthropic style ``{"type": "image", "source": {...}}`` where ``source`` + is either ``{"type": "base64", "media_type": ..., "data": ...}`` (→ + :class:`Blob`) or ``{"type": "url", "url": ...}`` (→ :class:`Uri`). + """ + block_type = item.get("type") + if block_type == "image_url": + image_url = item.get("image_url") + url: str | None = None + if isinstance(image_url, str): + url = image_url + elif isinstance(image_url, dict): + image_url_dict = cast(dict[str, Any], image_url) + raw_url = image_url_dict.get("url") + url = raw_url if isinstance(raw_url, str) else None + if not url: + return None + return image_from_url(url) + if block_type == "image": + source = item.get("source") + if not isinstance(source, dict): + return None + source_dict = cast(dict[str, Any], source) + source_type = source_dict.get("type") + if source_type == "base64": + data = source_dict.get("data") + if not isinstance(data, str): + return None + decoded = decode_base64(data) + if decoded is None: + return None + media_type = source_dict.get("media_type") + return Blob( + mime_type=( + media_type if isinstance(media_type, str) else None + ), + modality="image", + content=decoded, + ) + if source_type == "url": + source_url = source_dict.get("url") + if isinstance(source_url, str) and source_url: + return image_from_url(source_url) + return None + + def _content_to_parts( content: str | list[str | dict[str, Any]], ) -> list[MessagePart]: @@ -109,6 +164,10 @@ def _content_to_parts( ) if isinstance(reasoning_value, str) and reasoning_value: parts.append(Reasoning(content=reasoning_value)) + elif block_type in ("image_url", "image"): + media = _media_part(item) + if media is not None: + parts.append(media) return parts @@ -187,7 +246,11 @@ def _message_parts(message: BaseMessage) -> list[MessagePart]: def to_input_messages( messages: Iterable[Any], ) -> list[InputMessage]: - """Convert LangChain messages into spec-conformant ``InputMessage`` s.""" + """Convert LangChain messages into spec-conformant ``InputMessage`` s. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). + """ try: normalized_messages: Iterable[BaseMessage] = convert_to_messages( list(messages) @@ -216,6 +279,9 @@ def to_output_messages( as ``gen_ai.output.messages``. Tool execution results belong on the *input* side of the next inference call, not the output side of the previous one. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ result: list[OutputMessage] = [] for message in messages: @@ -295,6 +361,9 @@ def make_input_message(data: Any) -> list[InputMessage]: When no ``messages`` key exists (common in LangGraph state dicts), the remaining state fields are serialized as JSON and emitted as a single user-role :class:`Text` part. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ if not isinstance(data, dict): return [] @@ -329,6 +398,9 @@ def make_output_message(data: Any) -> list[OutputMessage]: empty: the underlying per-LLM-call finish reasons are recorded on child inference spans, and util-genai filters empty values out of ``gen_ai.response.finish_reasons``. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ if not isinstance(data, dict): return [] @@ -349,6 +421,9 @@ def make_last_output_message(data: Any) -> list[OutputMessage]: For Workflow and AgentInvocation spans, the final AI message best represents the actual output. Intermediate AI messages (e.g., tool-call decisions) are already captured in child LLM invocation spans. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ all_messages = make_output_message(data) if all_messages: diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml new file mode 100644 index 000000000..935890c3b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml @@ -0,0 +1,48 @@ +# TODO: this is generated by AI, re-record +# against the live Anthropic API once an ANTHROPIC_API_KEY is available. +interactions: +- request: + body: |- + {"model": "claude-sonnet-4-5", "max_tokens": 1024, "messages": [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg=="}}]}], "temperature": 0.1} + headers: + Content-Type: + - application/json + User-Agent: + - !!binary | + QW50aHJvcGljL1B5dGhvbiAxLjAuMA== + x-api-key: + - test_key + anthropic-version: + - '2023-06-01' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_01MultimodalImagePlaceholder", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "text", + "text": "This is a tiny 1x1 pixel PNG image." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 16, + "output_tokens": 12 + } + } + headers: + Content-Type: + - application/json + Date: + - Thu, 04 Sep 2025 20:00:58 GMT + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml new file mode 100644 index 000000000..92a562269 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml @@ -0,0 +1,230 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": [ + { + "type": "text", + "text": "What is in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" + } + } + ], + "role": "user" + } + ], + "model": "gpt-4o", + "max_completion_tokens": 100, + "stream": false, + "temperature": 0.1 + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '406' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.48.0 + x-stainless-arch: + - other:amd64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Windows + x-stainless-package-version: + - 2.48.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "choices": [ + { + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "protected_material_code": { + "detected": false, + "filtered": false + }, + "protected_material_text": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "This image consists of a yellow square centered on a blue background.", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 1785281713, + "id": "chatcmpl-E6lcHo1HleCxFE1A5235OXq03S1Jc", + "model": "gpt-4o-2024-11-20", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "jailbreak": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + } + } + ], + "service_tier": "default", + "system_fingerprint": "fp_91d870f097", + "usage": { + "completion_tokens": 14, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "latency_checkpoint": { + "engine_tbt_ms": 10, + "engine_ttft_ms": 48, + "engine_ttlt_ms": 186, + "pre_inference_ms": 127, + "service_tbt_ms": 11, + "service_ttft_ms": 477, + "service_ttlt_ms": 617, + "total_duration_ms": 497, + "user_visible_ttft_ms": 350 + }, + "prompt_tokens": 223, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 237 + } + } + headers: + Set-Cookie: test_set_cookie + apim-request-id: + - 5761d0a9-ee6a-488a-9878-ddea53ab04ea + azureai-fe-is-streaming: + - 'False' + azureai-fe-requested-service-tier: + - PayGo + azureai-fe-requested-zone: + - hot + azureml-model-session: + - d20260721052918-9061a802 + content-length: + - '1523' + content-type: + - application/json + date: + - Tue, 28 Jul 2026 23:35:13 GMT + openai-organization: test_openai_org_id + openai-project: test_openai_project_id + skip-error-remapping: + - 'true' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-ms-client-request-id: + - Not-Set + x-ms-is-spilled-over: + - 'false' + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US 2 + x-ms-served-model: + - gpt-4o-2024-11-20 + x-ratelimit-abusepenalty-active: + - 'False' + x-ratelimit-key: + - gpt-4o + x-ratelimit-limit-requests: + - '600' + x-ratelimit-limit-tokens: + - '100000' + x-ratelimit-remaining-requests: + - '599' + x-ratelimit-remaining-tokens: + - '99012' + x-ratelimit-renewalperiod-requests: + - '10' + x-ratelimit-renewalperiod-tokens: + - '60' + x-ratelimit-reset-requests: + - '0' + x-ratelimit-reset-tokens: + - '0' + x-request-id: + - 5761d0a9-ee6a-488a-9878-ddea53ab04ea + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/multimodal.py new file mode 100644 index 000000000..6008ac0e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/multimodal.py @@ -0,0 +1,218 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios: langchain multimodal chat. + +Exercises an inline base64 image content block through both ChatOpenAI (an +``image_url`` data URI) and ChatAnthropic (an ``image`` block with a +``source.base64`` payload), asserting the bytes round-trip onto the input +message as an image ``Blob`` part (``type == "blob"``, +``modality == "image"``). +""" + +from __future__ import annotations + +import json +import os +from typing import Any +from unittest import mock + +from langchain_anthropic import ChatAnthropic +from langchain_core.messages import HumanMessage +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.genai.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + +# A tiny valid PNG, base64-encoded. Pinned to the recorded cassettes' requests. +_REAL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3X" + "QQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAg" + "ICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" +) + + +class OpenAIMultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + # langchain can't populate server.address on chat spans. + expected_violations = ( + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("OPENAI_API_KEY") + else {"OPENAI_API_KEY": "test_openai_api_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + llm = ChatOpenAI( + model="gpt-4o", + temperature=0.1, + max_tokens=100, + ) + messages = [ + HumanMessage( + content=[ + { + "type": "text", + "text": "What is in this image?", + }, + { + "type": "image_url", + "image_url": { + "url": ( + "data:image/png;base64," + f"{_REAL_PNG_B64}" + ) + }, + }, + ] + ), + ] + with vcr.use_cassette( + "test_chat_openai_multimodal_image_llm_call.yaml" + ): + llm.invoke(messages) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + _assert_input_image_blob(report) + + +class AnthropicMultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + # langchain can't populate server.address on chat spans. + expected_violations = ( + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("ANTHROPIC_API_KEY") + else {"ANTHROPIC_API_KEY": "test_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + llm = ChatAnthropic( + model="claude-sonnet-4-5", + temperature=0.1, + max_tokens=1024, + ) + messages = [ + HumanMessage( + content=[ + { + "type": "text", + "text": "What is in this image?", + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + }, + ] + ), + ] + with vcr.use_cassette( + "test_chat_anthropic_multimodal_image_llm_call.yaml" + ): + llm.invoke(messages) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + _assert_input_image_blob(report) + + +def _assert_input_image_blob(report: LiveCheckReport) -> None: + # Lib-specific: weaver validates each part's *shape*, but not that an + # inline image actually round-tripped. Assert the base64 image landed + # on an input message as an image blob part. + chat_spans = [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and _attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + assert chat_spans, "no chat span emitted" + + input_parts = { + (t, m) + for span in chat_spans + for t, m in _part_fields(_attr(span, "gen_ai.input.messages")) + } + assert ("blob", "image") in input_parts, ( + f"expected an image blob part on an input message, saw {input_parts}" + ) + + +def _attr(span: dict[str, Any], name: str) -> Any: + for attr in span["attributes"]: + if attr["name"] == name: + return attr["value"] + return None + + +def _part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + # gen_ai.{input,output}.messages is a JSON string of + # [{"role": ..., "parts": [{"type": ..., "modality": ...}]}]. Keep + # modality so image/audio/video are distinguishable on blob parts. + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py index 95597a46e..d5c6d250a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py @@ -99,6 +99,16 @@ def fixture_gemini(): yield llm +@pytest.fixture(scope="function", name="chat_openai_vision") +def fixture_chat_openai_image(): + llm = ChatOpenAI( + model="gpt-4o", + temperature=0.1, + max_tokens=100, + ) + yield llm + + @pytest.fixture(scope="function") def start_instrumentation( tracer_provider, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt index fc4c7d834..fa900119b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt @@ -26,4 +26,6 @@ langchain-openai==0.2.0 langchain-aws==0.2.2 langchain-google-genai==2.0.0 langchain-anthropic==0.3.0 -boto3==1.37.0 \ No newline at end of file +boto3==1.37.0 + +-e ./util/opentelemetry-util-genai \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 1632c21f3..a8748bc59 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -8,6 +8,7 @@ the callback-handler logic and the invocation-manager bookkeeping. """ +import base64 import uuid from unittest import mock @@ -21,6 +22,7 @@ ) from opentelemetry.instrumentation.genai.langchain.utils import ( _legacy_function_call_request, + _media_part, extract_token_details, make_input_message, make_last_output_message, @@ -36,10 +38,12 @@ WorkflowInvocation, ) from opentelemetry.util.genai.types import ( + Blob, InputMessage, OutputMessage, Text, ToolCallRequest, + Uri, ) # --------------------------------------------------------------------------- @@ -1764,3 +1768,228 @@ def test_empty_header_value_ignored(self): handler.on_llm_end(response=response, run_id=run_id) assert llm_inv.response_model_name == "gpt-4o" + + +# --------------------------------------------------------------------------- +# utils._media_part - LangChain multimodal image block parsing +# --------------------------------------------------------------------------- + +_REAL_PNG_BYTES = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00" + b"\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc" + b"\xf8\xcf\xc0\xf0\x1f\x00\x05\x05\x02\x00\xa1\r\xf7\xdf\x00\x00\x00" + b"\x00IEND\xaeB`\x82" +) +_REAL_PNG_B64 = base64.b64encode(_REAL_PNG_BYTES).decode("ascii") + + +def test_media_part_openai_image_url_dict(): + item = { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/a.png" + + +def test_media_part_openai_image_url_string(): + item = {"type": "image_url", "image_url": "https://example.com/b.png"} + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/b.png" + + +def test_media_part_anthropic_base64_source_returns_blob(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "R0lGODlh", + }, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == b"GIF89a" + + +def test_media_part_anthropic_base64_source_without_media_type(): + item = { + "type": "image", + "source": {"type": "base64", "data": "QUJD"}, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type is None + assert part.content == b"ABC" + + +def test_media_part_base64_source_decodes_real_png(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_anthropic_url_source_returns_uri(): + item = { + "type": "image", + "source": {"type": "url", "url": "https://example.com/c.png"}, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/c.png" + + +def test_media_part_unrecognized_returns_none(): + assert _media_part({"type": "text", "text": "hi"}) is None + assert _media_part({"type": "image_url", "image_url": {}}) is None + assert ( + _media_part({"type": "image_url", "image_url": {"url": 123}}) is None + ) + assert _media_part({"type": "image", "source": "nope"}) is None + assert ( + _media_part({"type": "image", "source": {"type": "base64", "data": 5}}) + is None + ) + assert ( + _media_part({"type": "image", "source": {"type": "url", "url": ""}}) + is None + ) + + +def test_media_part_malformed_base64_returns_none(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "not!valid!base64!", + }, + } + assert _media_part(item) is None + + +def test_media_part_openai_real_png_data_uri_returns_blob(): + item = { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_anthropic_real_png_source_returns_blob(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + } + part = _media_part(item) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_anthropic_real_png_corrupted_base64_returns_none(): + corrupted = _REAL_PNG_B64[:10] + "@@@@" + _REAL_PNG_B64[14:] + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": corrupted, + }, + } + assert _media_part(item) is None + + +def test_media_part_real_png_url_source_returns_uri(): + item = { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/real-image.png", + }, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/real-image.png" + + +def test_to_input_messages_extracts_image_part(): + image_url = "data:image/jpeg;base64,QUJD" + content = [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": image_url}}, + ] + messages = to_input_messages([HumanMessage(content=content)]) + assert len(messages) == 1 + parts = messages[0].parts + + assert any(isinstance(p, Blob) for p in parts) + blob = next(p for p in parts if isinstance(p, Blob)) + assert blob.mime_type == "image/jpeg" + assert blob.content == b"ABC" + + +def test_on_chat_model_start_skips_input_messages_when_content_disabled(): + """Gating happens in the callback handler, not inside the utils.""" + run_id = _run_id() + handler, telemetry, llm_inv = _make_handler_with_llm_invocation(run_id) + telemetry.should_capture_content.return_value = False + + content = [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + }, + ] + handler.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content=content)]], + run_id=_run_id(), + invocation_params={"model_name": "gpt-4o"}, + ) + + assert llm_inv.input_messages == [] + + +def test_on_chat_model_start_captures_input_messages_when_content_enabled(): + run_id = _run_id() + handler, telemetry, llm_inv = _make_handler_with_llm_invocation(run_id) + telemetry.should_capture_content.return_value = True + + content = [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + }, + ] + handler.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content=content)]], + run_id=_run_id(), + invocation_params={"model_name": "gpt-4o"}, + ) + + parts = llm_inv.input_messages[0].parts + assert any(isinstance(p, Text) for p in parts) + blob = next(p for p in parts if isinstance(p, Blob)) + assert blob.content == _REAL_PNG_BYTES diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py index 569392419..af97ede95 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py @@ -22,6 +22,10 @@ from .conformance.agent import AgentScenario from .conformance.inference import InferenceScenario +from .conformance.multimodal import ( + AnthropicMultimodalScenario, + OpenAIMultimodalScenario, +) from .conformance.retrieval import RetrievalScenario from .conformance.tool_calling import ToolCallingScenario from .conformance.workflow import WorkflowScenario @@ -31,6 +35,8 @@ "scenario", [ InferenceScenario(), + OpenAIMultimodalScenario(), + AnthropicMultimodalScenario(), AgentScenario(), ToolCallingScenario(), WorkflowScenario(), diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 8529a061b..3e079acb8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -1,6 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +import base64 from importlib.metadata import version as _pkg_version from typing import Optional @@ -14,6 +15,9 @@ from langchain_core.tools import tool from openai import AuthenticationError +from opentelemetry.instrumentation.genai.langchain import ( + LangChainInstrumentor, +) from opentelemetry.instrumentation.genai.langchain.utils import ( to_input_messages, ) @@ -24,6 +28,7 @@ from opentelemetry.semconv._incubating.attributes import gen_ai_attributes from opentelemetry.semconv._incubating.metrics import gen_ai_metrics from opentelemetry.semconv.attributes import error_attributes +from opentelemetry.test_util_genai.instrumentor import instrument def _openai_cassette_name(model, base: str) -> str: @@ -56,6 +61,13 @@ def _langchain_openai_version() -> tuple: # cannot hold on those versions. _supports_reasoning_token_details = _langchain_openai_version() >= (0, 2, 1) +_REAL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3X" + "QQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAg" + "ICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" +) +_REAL_PNG_BYTES = base64.b64decode(_REAL_PNG_B64) + # span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py @pytest.mark.parametrize( @@ -66,28 +78,32 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call( span_exporter, metric_reader, log_exporter, - start_instrumentation, + tracer_provider, + meter_provider, + logger_provider, chat_openai_gpt_3_5_turbo_model, - monkeypatch, capture_content, vcr, ): - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content - ) - messages = [ SystemMessage(content="You are a helpful assistant!"), HumanMessage(content="What is the capital of France?"), ] - with vcr.use_cassette( - _openai_cassette_name( - chat_openai_gpt_3_5_turbo_model, - "test_chat_openai_gpt_3_5_turbo_model_llm_call", - ) + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture=capture_content, ): - response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + with vcr.use_cassette( + _openai_cassette_name( + chat_openai_gpt_3_5_turbo_model, + "test_chat_openai_gpt_3_5_turbo_model_llm_call", + ) + ): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) assert response.content == "The capital of France is Paris." # verify spans @@ -136,33 +152,37 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( span_exporter, metric_reader, log_exporter, - start_instrumentation, + tracer_provider, + meter_provider, + logger_provider, chat_openai_gpt_3_5_turbo_model, - monkeypatch, capture_content, vcr, ): - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content - ) - messages = [ SystemMessage(content="You are a helpful assistant!"), HumanMessage(content="What is the capital of France?"), ] response = None - try: - with vcr.use_cassette( - _openai_cassette_name( - chat_openai_gpt_3_5_turbo_model, - "test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error", - ) - ): - response = chat_openai_gpt_3_5_turbo_model.invoke(messages) - except Exception as e: - # For this test, to get error, cassettes were recorded with no OPENAI_API_KEY, so an error is expected here. - assert isinstance(e, AuthenticationError) + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture=capture_content, + ): + try: + with vcr.use_cassette( + _openai_cassette_name( + chat_openai_gpt_3_5_turbo_model, + "test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error", + ) + ): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + except Exception as e: + # For this test, to get error, cassettes were recorded with no OPENAI_API_KEY, so an error is expected here. + assert isinstance(e, AuthenticationError) assert response is None @@ -196,6 +216,78 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( assert len(logs) == 0 +def test_chat_openai_multimodal_image_llm_call( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + chat_openai_vision, + monkeypatch, + vcr, +): + """End-to-end: an OpenAI ``image_url`` content block is captured as an + image ``Blob`` part in ``gen_ai.input.messages``.""" + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{_REAL_PNG_B64}" + }, + }, + ] + ), + ] + + payload = chat_openai_vision._get_request_payload(messages, stop=None) + if "n" in payload: + pytest.skip( + "langchain-openai < 1.0 sends a different request body " + "(explicit n/temperature); only the modern cassette is recorded" + ) + + # The content-capture flag is cached when the handler is built at + # ``instrument()`` time, so set it before instrumenting (not after). + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + with vcr.use_cassette( + "test_chat_openai_multimodal_image_llm_call.yaml" + ): + chat_openai_vision.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "gpt-4o" + ) + + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + finally: + instrumentor.uninstrument() + + # span_exporter, start_instrumentation, us_amazon_nova_lite_v1_0 are coming from fixtures defined in conftest.py def test_us_amazon_nova_lite_v1_0_bedrock_llm_call( span_exporter, start_instrumentation, us_amazon_nova_lite_v1_0, vcr @@ -372,6 +464,70 @@ def test_chat_openai_legacy_function_call( assert '"location"' in tool_definitions +@pytest.mark.vcr() +def test_chat_anthropic_multimodal_image_llm_call( + span_exporter, + tracer_provider, + meter_provider, + logger_provider, + chat_anthropic_claude_sonnet, + monkeypatch, +): + """End-to-end: an Anthropic ``image`` content block is captured as an + image ``Blob`` part in ``gen_ai.input.messages``.""" + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + }, + ] + ), + ] + + # The content-capture flag is cached when the handler is built at + # ``instrument()`` time, so set it before instrumenting (not after). + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + chat_anthropic_claude_sonnet.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "claude-sonnet-4-5" + ) + + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + finally: + instrumentor.uninstrument() + + # span_exporter, start_instrumentation, gemini are coming from fixtures defined in conftest.py def test_gemini(span_exporter, start_instrumentation, gemini, vcr): messages = [ diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index a78b46b7b..cd3f08e74 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -4,7 +4,7 @@ import json import logging import os -from base64 import b64encode +from base64 import b64decode, b64encode from functools import partial from typing import Any @@ -12,7 +12,12 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, OTEL_INSTRUMENTATION_GENAI_EMIT_EVENT, ) -from opentelemetry.util.genai.types import ContentCapturingMode +from opentelemetry.util.genai.types import ( + Blob, + ContentCapturingMode, + MessagePart, + Uri, +) logger = logging.getLogger(__name__) @@ -36,6 +41,46 @@ def get_content_capturing_mode() -> ContentCapturingMode: return ContentCapturingMode.NO_CONTENT +def decode_base64(data: str) -> bytes | None: + """Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). + """ + try: + return b64decode("".join(data.split()), validate=True) + except Exception: # pylint: disable=broad-exception-caught + return None + + +def image_from_url(url: str, *, modality: str = "image") -> MessagePart | None: + """Return a media part for an image ``url``. + + A ``data:;base64,`` URL is decoded into a + :class:`~opentelemetry.util.genai.types.Blob`; a ``data:`` URL without + base64 encoding keeps its raw payload bytes; any other URL becomes a + :class:`~opentelemetry.util.genai.types.Uri`. Shared by instrumentations + that parse provider image blocks. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). + """ + if url.startswith("data:"): + header, _, payload = url[len("data:") :].partition(",") + mime_type = header.split(";", 1)[0] or None + if ";base64" in header: + decoded = decode_base64(payload) + if decoded is None: + return None + content = decoded + else: + content = payload.encode("utf-8") + return Blob( + mime_type=mime_type, + modality=modality, + content=content, + ) + return Uri(mime_type=None, modality=modality, uri=url) + + def is_experimental_mode() -> bool: """ Kept for backwards compatibility. The utils in this library only support the experimental mode sem convs now. diff --git a/util/opentelemetry-util-genai/tests/test_utils.py b/util/opentelemetry-util-genai/tests/test_utils.py index 127e9abae..977dcd255 100644 --- a/util/opentelemetry-util-genai/tests/test_utils.py +++ b/util/opentelemetry-util-genai/tests/test_utils.py @@ -1,6 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +import base64 import json import os import unittest @@ -34,14 +35,18 @@ get_telemetry_handler, ) from opentelemetry.util.genai.types import ( + Blob, ContentCapturingMode, InputMessage, MessagePart, OutputMessage, Text, + Uri, ) from opentelemetry.util.genai.utils import ( + decode_base64, get_content_capturing_mode, + image_from_url, should_capture_content_on_spans, should_emit_event, ) @@ -956,3 +961,112 @@ def test_inference_finish_does_not_duplicate_start_attributes(self): class AnyNonNone: def __eq__(self, other): return other is not None + + +_REAL_PNG_BYTES = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00" + b"\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc" + b"\xf8\xcf\xc0\xf0\x1f\x00\x05\x05\x02\x00\xa1\r\xf7\xdf\x00\x00\x00" + b"\x00IEND\xaeB`\x82" +) +_REAL_PNG_B64 = base64.b64encode(_REAL_PNG_BYTES).decode("ascii") + + +class TestMediaHelpers(unittest.TestCase): + """Tests for the shared ``decode_base64`` / ``image_from_url`` helpers. + + Both helpers unconditionally decode the payload they are given; callers are + expected to check whether content capture is enabled and skip calling them + when it is not. + """ + + # -- image_from_url -------------------------------------------------- + + def test_image_from_url_data_uri_returns_blob(self): + part = image_from_url("data:image/jpeg;base64,QUJD") + self.assertIsInstance(part, Blob) + self.assertEqual(part.mime_type, "image/jpeg") + self.assertEqual(part.modality, "image") + self.assertEqual(part.content, b"ABC") + + def test_image_from_url_http_returns_uri(self): + part = image_from_url("https://example.com/cat.png") + self.assertIsInstance(part, Uri) + self.assertEqual(part.uri, "https://example.com/cat.png") + self.assertEqual(part.modality, "image") + + def test_image_from_url_data_uri_without_base64_keeps_text_bytes(self): + part = image_from_url("data:text/plain,hello") + self.assertIsInstance(part, Blob) + self.assertEqual(part.mime_type, "text/plain") + self.assertEqual(part.content, b"hello") + + def test_image_from_url_data_uri_no_mime_type(self): + part = image_from_url("data:;base64,QUJD") + self.assertIsInstance(part, Blob) + self.assertIsNone(part.mime_type) + self.assertEqual(part.content, b"ABC") + + def test_image_from_url_data_uri_malformed_base64_returns_none(self): + part = image_from_url("data:image/png;base64,not!valid!base64!") + self.assertIsNone(part) + + def test_image_from_url_honours_modality_override(self): + part = image_from_url("data:audio/mp3;base64,QUJD", modality="audio") + self.assertIsInstance(part, Blob) + self.assertEqual(part.modality, "audio") + uri_part = image_from_url( + "https://example.com/a.mp3", modality="audio" + ) + self.assertIsInstance(uri_part, Uri) + self.assertEqual(uri_part.modality, "audio") + + # -- decode_base64 --------------------------------------------------- + + def test_decode_base64_valid_returns_bytes(self): + self.assertEqual(decode_base64("QUJD"), b"ABC") + + def test_decode_base64_valid_with_padding(self): + self.assertEqual(decode_base64("R0lGODlh"), b"GIF89a") + + def test_decode_base64_strips_whitespace_and_newlines(self): + self.assertEqual(decode_base64("QU\nJD"), b"ABC") + self.assertEqual(decode_base64(" QUJD "), b"ABC") + self.assertEqual(decode_base64("QU JD"), b"ABC") + + def test_decode_base64_malformed_returns_none(self): + self.assertIsNone(decode_base64("not!valid!base64!")) + self.assertIsNone(decode_base64("@@@@")) + self.assertIsNone(decode_base64("****")) + + def test_decode_base64_wrong_padding_returns_none(self): + # Correct base64 alphabet but invalid length/padding. + self.assertIsNone(decode_base64("QUJ")) + self.assertIsNone(decode_base64("QQ")) + + def test_decode_base64_empty_returns_empty_bytes(self): + self.assertEqual(decode_base64(""), b"") + + def test_decode_base64_real_image_round_trips(self): + self.assertEqual(decode_base64(_REAL_PNG_B64), _REAL_PNG_BYTES) + + @patch.dict( + os.environ, + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT"}, + ) + def test_decode_base64_ignores_content_capture_env(self): + # Gating is the caller's responsibility, so the helper never reads the + # content-capture environment variable itself. + self.assertEqual(decode_base64("QUJD"), b"ABC") + self.assertEqual(decode_base64(_REAL_PNG_B64), _REAL_PNG_BYTES) + + def test_image_from_url_real_png_data_uri_returns_blob(self): + part = image_from_url(f"data:image/png;base64,{_REAL_PNG_B64}") + self.assertIsInstance(part, Blob) + self.assertEqual(part.mime_type, "image/png") + self.assertEqual(part.content, _REAL_PNG_BYTES) + + def test_image_from_url_real_png_truncated_base64_returns_none(self): + corrupted = _REAL_PNG_B64[:-4] + "!!!!" + part = image_from_url(f"data:image/png;base64,{corrupted}") + self.assertIsNone(part)