From 8837fc48662feb3147b095824f2d50aecb0076d1 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 21 Jul 2026 12:03:38 -0700 Subject: [PATCH 01/14] Capture multimodal image content (OpenAI `image_url` and Anthropic `image` blocks) as `Blob`/`Uri` message parts. --- .../.changelog/x.added | 1 + .../instrumentation/genai/langchain/utils.py | 84 +++++++++++++++++++ .../tests/test_callback_handler.py | 4 + 3 files changed, 89 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/x.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/x.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/x.added new file mode 100644 index 000000000..2c4b09dc1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/x.added @@ -0,0 +1 @@ +(Openinference Migration: Langchain) - Capture multimodal image content (OpenAI ``image_url`` and Anthropic ``image`` blocks) as ``Blob``/``Uri`` message parts. diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index ef726f1ac..5b16fb920 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -4,6 +4,7 @@ from __future__ import annotations +import base64 import json from collections.abc import Iterable from typing import Any, cast @@ -19,6 +20,7 @@ gen_ai_attributes as GenAIAttributes, ) from opentelemetry.util.genai.types import ( + Blob, FunctionToolDefinition, InputMessage, MessagePart, @@ -28,6 +30,7 @@ ToolCallRequest, ToolCallResponse, ToolDefinition, + Uri, ) # Mapping from LangChain ``ls_provider`` metadata values to the well-known @@ -75,6 +78,83 @@ def _normalize_role(message: BaseMessage) -> str: return _ROLE_MAP.get(message.type, message.type) +def _decode_base64(data: str) -> Optional[bytes]: + try: + return base64.b64decode(data) + except Exception: # pylint: disable=broad-exception-caught + return None + + +def _media_part(item: dict[str, Any]) -> Optional[MessagePart]: + """Convert a LangChain multimodal image content block into a media part. + + Handles the two shapes LangChain chat models accept: + + - OpenAI style ``{"type": "image_url", "image_url": {"url": ...}}`` (or a + bare ``"image_url": "..."`` string). A ``data:;base64,`` + URL becomes a :class:`Blob`; any other URL becomes a :class:`Uri`. + - Anthropic style ``{"type": "image", "source": {...}}`` where ``source`` + is either ``{"type": "base64", "media_type": ..., "data": ...}`` (→ + :class:`Blob`) or ``{"type": "url", "url": ...}`` (→ :class:`Uri`). + """ + block_type = item.get("type") + if block_type == "image_url": + image_url = item.get("image_url") + url: Optional[str] = None + if isinstance(image_url, str): + url = image_url + elif isinstance(image_url, dict): + raw_url = image_url.get("url") + url = raw_url if isinstance(raw_url, str) else None + if not url: + return None + return _image_from_url(url) + if block_type == "image": + source = item.get("source") + if not isinstance(source, dict): + return None + source_type = source.get("type") + if source_type == "base64": + data = source.get("data") + if not isinstance(data, str): + return None + decoded = _decode_base64(data) + if decoded is None: + return None + media_type = source.get("media_type") + return Blob( + mime_type=media_type if isinstance(media_type, str) else None, + modality="image", + content=decoded, + ) + if source_type == "url": + url = source.get("url") + if isinstance(url, str) and url: + return _image_from_url(url) + return None + + +def _image_from_url(url: str) -> Optional[MessagePart]: + """Return a :class:`Blob` for a ``data:`` URL, else a :class:`Uri`.""" + + if url.startswith("data:"): + header, _, payload = url[len("data:") :].partition(",") + mime_type = header.split(";", 1)[0] or None + if ";base64" in header: + decoded = _decode_base64(payload) + if decoded is None: + return None + content = decoded + else: + content = payload.encode("utf-8") + return Blob( + mime_type=mime_type, + modality="image", + content=content, + ) + return Uri(mime_type=None, modality="image", uri=url) + + def _content_to_parts( content: str | list[str | dict[str, Any]], ) -> list[MessagePart]: @@ -109,6 +189,10 @@ def _content_to_parts( ) if isinstance(reasoning_value, str) and reasoning_value: parts.append(Reasoning(content=reasoning_value)) + elif block_type in ("image_url", "image"): + media = _media_part(item) + if media is not None: + parts.append(media) return parts diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 1632c21f3..ea5962a64 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -22,6 +22,8 @@ from opentelemetry.instrumentation.genai.langchain.utils import ( _legacy_function_call_request, extract_token_details, + _image_from_url, + _media_part, make_input_message, make_last_output_message, make_output_message, @@ -36,10 +38,12 @@ WorkflowInvocation, ) from opentelemetry.util.genai.types import ( + Blob, InputMessage, OutputMessage, Text, ToolCallRequest, + Uri, ) # --------------------------------------------------------------------------- From 41b35b639933747a1c4d90c44cc69aa43b86a39e Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 21 Jul 2026 13:00:00 -0700 Subject: [PATCH 02/14] Update CHANGELOG number --- .../.changelog/{x.added => 296.added} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/{x.added => 296.added} (100%) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/x.added b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added similarity index 100% rename from instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/x.added rename to instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/296.added From 5464667865ab617fd443eccd1a39712e75ab21e6 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 21 Jul 2026 13:13:43 -0700 Subject: [PATCH 03/14] Fix typecheck errors --- .../instrumentation/genai/langchain/utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index 5b16fb920..8da0fb9de 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -104,7 +104,8 @@ def _media_part(item: dict[str, Any]) -> Optional[MessagePart]: if isinstance(image_url, str): url = image_url elif isinstance(image_url, dict): - raw_url = image_url.get("url") + image_url_dict = cast(dict[str, Any], image_url) + raw_url = image_url_dict.get("url") url = raw_url if isinstance(raw_url, str) else None if not url: return None @@ -113,24 +114,25 @@ def _media_part(item: dict[str, Any]) -> Optional[MessagePart]: source = item.get("source") if not isinstance(source, dict): return None - source_type = source.get("type") + source_dict = cast(dict[str, Any], source) + source_type = source_dict.get("type") if source_type == "base64": - data = source.get("data") + data = source_dict.get("data") if not isinstance(data, str): return None decoded = _decode_base64(data) if decoded is None: return None - media_type = source.get("media_type") + media_type = source_dict.get("media_type") return Blob( mime_type=media_type if isinstance(media_type, str) else None, modality="image", content=decoded, ) if source_type == "url": - url = source.get("url") - if isinstance(url, str) and url: - return _image_from_url(url) + source_url = source_dict.get("url") + if isinstance(source_url, str) and source_url: + return _image_from_url(source_url) return None From 8ef04a2bd415ea293508ff63832987672e449d4e Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 21 Jul 2026 14:58:56 -0700 Subject: [PATCH 04/14] Address feedback --- .../src/opentelemetry/instrumentation/genai/langchain/utils.py | 2 +- .../tests/test_callback_handler.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index 8da0fb9de..5454c3422 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -80,7 +80,7 @@ def _normalize_role(message: BaseMessage) -> str: def _decode_base64(data: str) -> Optional[bytes]: try: - return base64.b64decode(data) + return base64.b64decode("".join(data.split()), validate=True) except Exception: # pylint: disable=broad-exception-caught return None diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index ea5962a64..7e16a3510 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -8,6 +8,7 @@ the callback-handler logic and the invocation-manager bookkeeping. """ +import base64 import uuid from unittest import mock @@ -22,6 +23,7 @@ from opentelemetry.instrumentation.genai.langchain.utils import ( _legacy_function_call_request, extract_token_details, + _decode_base64, _image_from_url, _media_part, make_input_message, From 86522fa9227c5c61415257306c27b7b7411d2ba4 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 29 Jul 2026 08:58:17 -0700 Subject: [PATCH 05/14] Add recording tests --- ...t_anthropic_multimodal_image_llm_call.yaml | 48 ++++ ...chat_openai_multimodal_image_llm_call.yaml | 230 ++++++++++++++++++ .../tests/conftest.py | 10 + .../tests/test_callback_handler.py | 4 +- .../tests/test_llm_call.py | 117 +++++++++ 5 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml new file mode 100644 index 000000000..935890c3b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_anthropic_multimodal_image_llm_call.yaml @@ -0,0 +1,48 @@ +# TODO: this is generated by AI, re-record +# against the live Anthropic API once an ANTHROPIC_API_KEY is available. +interactions: +- request: + body: |- + {"model": "claude-sonnet-4-5", "max_tokens": 1024, "messages": [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg=="}}]}], "temperature": 0.1} + headers: + Content-Type: + - application/json + User-Agent: + - !!binary | + QW50aHJvcGljL1B5dGhvbiAxLjAuMA== + x-api-key: + - test_key + anthropic-version: + - '2023-06-01' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "id": "msg_01MultimodalImagePlaceholder", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + { + "type": "text", + "text": "This is a tiny 1x1 pixel PNG image." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 16, + "output_tokens": 12 + } + } + headers: + Content-Type: + - application/json + Date: + - Thu, 04 Sep 2025 20:00:58 GMT + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml new file mode 100644 index 000000000..92a562269 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/cassettes/test_chat_openai_multimodal_image_llm_call.yaml @@ -0,0 +1,230 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": [ + { + "type": "text", + "text": "What is in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3XQQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAgICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" + } + } + ], + "role": "user" + } + ], + "model": "gpt-4o", + "max_completion_tokens": 100, + "stream": false, + "temperature": 0.1 + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '406' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.48.0 + x-stainless-arch: + - other:amd64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Windows + x-stainless-package-version: + - 2.48.0 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "choices": [ + { + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "protected_material_code": { + "detected": false, + "filtered": false + }, + "protected_material_text": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "This image consists of a yellow square centered on a blue background.", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 1785281713, + "id": "chatcmpl-E6lcHo1HleCxFE1A5235OXq03S1Jc", + "model": "gpt-4o-2024-11-20", + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": { + "filtered": false, + "severity": "safe" + }, + "jailbreak": { + "detected": false, + "filtered": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": false, + "severity": "safe" + } + } + } + ], + "service_tier": "default", + "system_fingerprint": "fp_91d870f097", + "usage": { + "completion_tokens": 14, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "latency_checkpoint": { + "engine_tbt_ms": 10, + "engine_ttft_ms": 48, + "engine_ttlt_ms": 186, + "pre_inference_ms": 127, + "service_tbt_ms": 11, + "service_ttft_ms": 477, + "service_ttlt_ms": 617, + "total_duration_ms": 497, + "user_visible_ttft_ms": 350 + }, + "prompt_tokens": 223, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 237 + } + } + headers: + Set-Cookie: test_set_cookie + apim-request-id: + - 5761d0a9-ee6a-488a-9878-ddea53ab04ea + azureai-fe-is-streaming: + - 'False' + azureai-fe-requested-service-tier: + - PayGo + azureai-fe-requested-zone: + - hot + azureml-model-session: + - d20260721052918-9061a802 + content-length: + - '1523' + content-type: + - application/json + date: + - Tue, 28 Jul 2026 23:35:13 GMT + openai-organization: test_openai_org_id + openai-project: test_openai_project_id + skip-error-remapping: + - 'true' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-accel-buffering: + - 'no' + x-content-type-options: + - nosniff + x-ms-client-request-id: + - Not-Set + x-ms-is-spilled-over: + - 'false' + x-ms-rai-invoked: + - 'true' + x-ms-region: + - East US 2 + x-ms-served-model: + - gpt-4o-2024-11-20 + x-ratelimit-abusepenalty-active: + - 'False' + x-ratelimit-key: + - gpt-4o + x-ratelimit-limit-requests: + - '600' + x-ratelimit-limit-tokens: + - '100000' + x-ratelimit-remaining-requests: + - '599' + x-ratelimit-remaining-tokens: + - '99012' + x-ratelimit-renewalperiod-requests: + - '10' + x-ratelimit-renewalperiod-tokens: + - '60' + x-ratelimit-reset-requests: + - '0' + x-ratelimit-reset-tokens: + - '0' + x-request-id: + - 5761d0a9-ee6a-488a-9878-ddea53ab04ea + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py index 95597a46e..d5c6d250a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conftest.py @@ -99,6 +99,16 @@ def fixture_gemini(): yield llm +@pytest.fixture(scope="function", name="chat_openai_vision") +def fixture_chat_openai_image(): + llm = ChatOpenAI( + model="gpt-4o", + temperature=0.1, + max_tokens=100, + ) + yield llm + + @pytest.fixture(scope="function") def start_instrumentation( tracer_provider, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index 7e16a3510..ebccfe682 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -21,11 +21,11 @@ OpenTelemetryLangChainCallbackHandler, ) from opentelemetry.instrumentation.genai.langchain.utils import ( - _legacy_function_call_request, - extract_token_details, _decode_base64, _image_from_url, + _legacy_function_call_request, _media_part, + extract_token_details, make_input_message, make_last_output_message, make_output_message, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 8529a061b..07fa5f44f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -1,6 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +import base64 from importlib.metadata import version as _pkg_version from typing import Optional @@ -56,6 +57,13 @@ def _langchain_openai_version() -> tuple: # cannot hold on those versions. _supports_reasoning_token_details = _langchain_openai_version() >= (0, 2, 1) +_REAL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3X" + "QQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAg" + "ICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" +) +_REAL_PNG_BYTES = base64.b64decode(_REAL_PNG_B64) + # span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py @pytest.mark.parametrize( @@ -196,6 +204,63 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( assert len(logs) == 0 +def test_chat_openai_multimodal_image_llm_call( + span_exporter, + start_instrumentation, + chat_openai_vision, + monkeypatch, + vcr, +): + """End-to-end: an OpenAI ``image_url`` content block is captured as an + image ``Blob`` part in ``gen_ai.input.messages``.""" + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{_REAL_PNG_B64}" + }, + }, + ] + ), + ] + + payload = chat_openai_vision._get_request_payload(messages, stop=None) + if "n" in payload: + pytest.skip( + "langchain-openai < 1.0 sends a different request body " + "(explicit n/temperature); only the modern cassette is recorded" + ) + with vcr.use_cassette("test_chat_openai_multimodal_image_llm_call.yaml"): + chat_openai_vision.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "gpt-4o" + ) + + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + + # span_exporter, start_instrumentation, us_amazon_nova_lite_v1_0 are coming from fixtures defined in conftest.py def test_us_amazon_nova_lite_v1_0_bedrock_llm_call( span_exporter, start_instrumentation, us_amazon_nova_lite_v1_0, vcr @@ -372,6 +437,58 @@ def test_chat_openai_legacy_function_call( assert '"location"' in tool_definitions +@pytest.mark.vcr() +def test_chat_anthropic_multimodal_image_llm_call( + span_exporter, + start_instrumentation, + chat_anthropic_claude_sonnet, + monkeypatch, +): + """End-to-end: an Anthropic ``image`` content block is captured as an + image ``Blob`` part in ``gen_ai.input.messages``.""" + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + }, + ] + ), + ] + + chat_anthropic_claude_sonnet.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "claude-sonnet-4-5" + ) + + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + + # span_exporter, start_instrumentation, gemini are coming from fixtures defined in conftest.py def test_gemini(span_exporter, start_instrumentation, gemini, vcr): messages = [ From 501f7a3f3a53e3068bd44117b9a13bc48e5afa04 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Thu, 30 Jul 2026 16:44:23 -0700 Subject: [PATCH 06/14] Address feedback --- .../opentelemetry/instrumentation/genai/langchain/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index 5454c3422..001ba6eba 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -21,6 +21,7 @@ ) from opentelemetry.util.genai.types import ( Blob, + ContentCapturingMode, FunctionToolDefinition, InputMessage, MessagePart, @@ -32,6 +33,7 @@ ToolDefinition, Uri, ) +from opentelemetry.util.genai.utils import get_content_capturing_mode # Mapping from LangChain ``ls_provider`` metadata values to the well-known # ``gen_ai.provider.name`` values defined by the GenAI semantic conventions. @@ -79,6 +81,10 @@ def _normalize_role(message: BaseMessage) -> str: def _decode_base64(data: str) -> Optional[bytes]: + # Skip the decode entirely when message content is not being captured; + # the resulting bytes would never be emitted under ``NO_CONTENT``. + if get_content_capturing_mode() is ContentCapturingMode.NO_CONTENT: + return None try: return base64.b64decode("".join(data.split()), validate=True) except Exception: # pylint: disable=broad-exception-caught From 80774cf1c4a9ddd9037bc8b23f82b7038f8fbaaa Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 5 Aug 2026 16:04:46 -0700 Subject: [PATCH 07/14] Fix format per new ruff version --- .../instrumentation/genai/langchain/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index 001ba6eba..d0a0dbf7d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -80,7 +80,7 @@ def _normalize_role(message: BaseMessage) -> str: return _ROLE_MAP.get(message.type, message.type) -def _decode_base64(data: str) -> Optional[bytes]: +def _decode_base64(data: str) -> bytes | None: # Skip the decode entirely when message content is not being captured; # the resulting bytes would never be emitted under ``NO_CONTENT``. if get_content_capturing_mode() is ContentCapturingMode.NO_CONTENT: @@ -91,7 +91,7 @@ def _decode_base64(data: str) -> Optional[bytes]: return None -def _media_part(item: dict[str, Any]) -> Optional[MessagePart]: +def _media_part(item: dict[str, Any]) -> MessagePart | None: """Convert a LangChain multimodal image content block into a media part. Handles the two shapes LangChain chat models accept: @@ -106,7 +106,7 @@ def _media_part(item: dict[str, Any]) -> Optional[MessagePart]: block_type = item.get("type") if block_type == "image_url": image_url = item.get("image_url") - url: Optional[str] = None + url: str | None = None if isinstance(image_url, str): url = image_url elif isinstance(image_url, dict): @@ -142,7 +142,7 @@ def _media_part(item: dict[str, Any]) -> Optional[MessagePart]: return None -def _image_from_url(url: str) -> Optional[MessagePart]: +def _image_from_url(url: str) -> MessagePart | None: """Return a :class:`Blob` for a ``data:`` URL, else a :class:`Uri`.""" if url.startswith("data:"): From c2a941fb7a83fca23f93a28384db57a59f13b6cc Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 10 Aug 2026 15:59:47 -0700 Subject: [PATCH 08/14] Address feedback --- .../genai/langchain/callback_handler.py | 18 +- .../instrumentation/genai/langchain/utils.py | 102 ++++---- .../tests/conformance/multimodal.py | 218 +++++++++++++++++ .../tests/test_callback_handler.py | 228 +++++++++++++++++- .../tests/test_conformance.py | 6 + .../src/opentelemetry/util/genai/utils.py | 46 +++- .../tests/test_utils.py | 146 +++++++++++ 7 files changed, 700 insertions(+), 64 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/multimodal.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index c41f3a386..12e7e35c1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -73,7 +73,7 @@ def on_chain_start( operation = classify_chain_run( serialized, metadata, kwargs, parent_run_id ) - + capture_content = self._telemetry_handler.should_capture_content() if operation == OperationName.INVOKE_WORKFLOW: workflow_name = kwargs.get("name") or serialized.get("name") workflow_name_override = ( @@ -82,7 +82,9 @@ def on_chain_start( workflow = self._telemetry_handler.workflow( name=workflow_name_override or workflow_name ) - workflow.input_messages = make_input_message(inputs) + workflow.input_messages = make_input_message( + inputs, capture_content + ) self._invocation_manager.add_invocation_state( run_id, parent_run_id, workflow ) @@ -107,7 +109,9 @@ def on_chain_start( agent = self._telemetry_handler.invoke_local_agent( agent_name=suggested_agent_name, ) - agent.input_messages = make_input_message(inputs) + agent.input_messages = make_input_message( + inputs, capture_content + ) if metadata: agent.agent_id = metadata.get("agent_id") @@ -162,7 +166,10 @@ def on_chain_end( self._invocation_manager.delete_invocation_state(run_id) return - invocation.output_messages = make_last_output_message(outputs) + capture_content = self._telemetry_handler.should_capture_content() + invocation.output_messages = make_last_output_message( + outputs, capture_content + ) invocation.stop() @@ -270,7 +277,8 @@ def on_chat_model_start( # :func:`to_input_messages` produce spec-conformant ``InputMessage`` s # with proper roles, tool-call requests, tool results, and reasoning. flattened: list[BaseMessage] = [msg for sub in messages for msg in sub] - input_messages = to_input_messages(flattened) + capture_content = self._telemetry_handler.should_capture_content() + input_messages = to_input_messages(flattened, capture_content) llm_invocation = self._telemetry_handler.inference( provider, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index d0a0dbf7d..e575b1c9a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -4,7 +4,6 @@ from __future__ import annotations -import base64 import json from collections.abc import Iterable from typing import Any, cast @@ -21,7 +20,6 @@ ) from opentelemetry.util.genai.types import ( Blob, - ContentCapturingMode, FunctionToolDefinition, InputMessage, MessagePart, @@ -31,9 +29,8 @@ ToolCallRequest, ToolCallResponse, ToolDefinition, - Uri, ) -from opentelemetry.util.genai.utils import get_content_capturing_mode +from opentelemetry.util.genai.utils import decode_base64, image_from_url # Mapping from LangChain ``ls_provider`` metadata values to the well-known # ``gen_ai.provider.name`` values defined by the GenAI semantic conventions. @@ -80,18 +77,9 @@ def _normalize_role(message: BaseMessage) -> str: return _ROLE_MAP.get(message.type, message.type) -def _decode_base64(data: str) -> bytes | None: - # Skip the decode entirely when message content is not being captured; - # the resulting bytes would never be emitted under ``NO_CONTENT``. - if get_content_capturing_mode() is ContentCapturingMode.NO_CONTENT: - return None - try: - return base64.b64decode("".join(data.split()), validate=True) - except Exception: # pylint: disable=broad-exception-caught - return None - - -def _media_part(item: dict[str, Any]) -> MessagePart | None: +def _media_part( + item: dict[str, Any], capture_content: bool = False +) -> MessagePart | None: """Convert a LangChain multimodal image content block into a media part. Handles the two shapes LangChain chat models accept: @@ -115,7 +103,7 @@ def _media_part(item: dict[str, Any]) -> MessagePart | None: url = raw_url if isinstance(raw_url, str) else None if not url: return None - return _image_from_url(url) + return image_from_url(url, capture_content=capture_content) if block_type == "image": source = item.get("source") if not isinstance(source, dict): @@ -126,45 +114,29 @@ def _media_part(item: dict[str, Any]) -> MessagePart | None: data = source_dict.get("data") if not isinstance(data, str): return None - decoded = _decode_base64(data) + decoded = decode_base64(data, capture_content) if decoded is None: return None media_type = source_dict.get("media_type") return Blob( - mime_type=media_type if isinstance(media_type, str) else None, + mime_type=( + media_type if isinstance(media_type, str) else None + ), modality="image", content=decoded, ) if source_type == "url": source_url = source_dict.get("url") if isinstance(source_url, str) and source_url: - return _image_from_url(source_url) + return image_from_url( + source_url, capture_content=capture_content + ) return None -def _image_from_url(url: str) -> MessagePart | None: - """Return a :class:`Blob` for a ``data:`` URL, else a :class:`Uri`.""" - - if url.startswith("data:"): - header, _, payload = url[len("data:") :].partition(",") - mime_type = header.split(";", 1)[0] or None - if ";base64" in header: - decoded = _decode_base64(payload) - if decoded is None: - return None - content = decoded - else: - content = payload.encode("utf-8") - return Blob( - mime_type=mime_type, - modality="image", - content=content, - ) - return Uri(mime_type=None, modality="image", uri=url) - - def _content_to_parts( content: str | list[str | dict[str, Any]], + capture_content: bool = False, ) -> list[MessagePart]: """Convert a LangChain message ``content`` payload into ``MessagePart`` s. @@ -198,7 +170,7 @@ def _content_to_parts( if isinstance(reasoning_value, str) and reasoning_value: parts.append(Reasoning(content=reasoning_value)) elif block_type in ("image_url", "image"): - media = _media_part(item) + media = _media_part(item, capture_content) if media is not None: parts.append(media) return parts @@ -231,14 +203,18 @@ def _legacy_function_call_request( return ToolCallRequest(arguments=arguments, name=name, id=None) -def _ai_message_parts(message: AIMessage) -> list[MessagePart]: +def _ai_message_parts( + message: AIMessage, capture_content: bool = False +) -> list[MessagePart]: """Build :class:`MessagePart` s for an :class:`AIMessage`. Includes any text/reasoning content followed by a :class:`ToolCallRequest` for each entry in ``message.tool_calls``, plus a legacy ``additional_kwargs['function_call']`` when present. """ - parts: list[MessagePart] = _content_to_parts(message.content) + parts: list[MessagePart] = _content_to_parts( + message.content, capture_content + ) for call in message.tool_calls: name = call["name"] if not name: @@ -268,16 +244,19 @@ def _tool_message_parts(message: ToolMessage) -> list[MessagePart]: ] -def _message_parts(message: BaseMessage) -> list[MessagePart]: +def _message_parts( + message: BaseMessage, capture_content: bool = False +) -> list[MessagePart]: if isinstance(message, ToolMessage): return _tool_message_parts(message) if isinstance(message, AIMessage): - return _ai_message_parts(message) - return _content_to_parts(message.content) + return _ai_message_parts(message, capture_content) + return _content_to_parts(message.content, capture_content) def to_input_messages( messages: Iterable[Any], + capture_content: bool = False, ) -> list[InputMessage]: """Convert LangChain messages into spec-conformant ``InputMessage`` s.""" try: @@ -290,7 +269,7 @@ def to_input_messages( ] result: list[InputMessage] = [] for message in normalized_messages: - parts = _message_parts(message) + parts = _message_parts(message, capture_content) if not parts: continue result.append(InputMessage(role=_normalize_role(message), parts=parts)) @@ -301,6 +280,7 @@ def to_output_messages( messages: Iterable[BaseMessage], *, finish_reason: str = "", + capture_content: bool = False, ) -> list[OutputMessage]: """Convert LangChain ``AIMessage`` instances into ``OutputMessage`` s. @@ -313,7 +293,7 @@ def to_output_messages( for message in messages: if not isinstance(message, AIMessage): continue - parts = _ai_message_parts(message) + parts = _ai_message_parts(message, capture_content) if not parts: continue result.append( @@ -375,7 +355,9 @@ def prepare_tool_definitions(tools: list[Any]) -> list[ToolDefinition] | None: return definitions or None -def make_input_message(data: Any) -> list[InputMessage]: +def make_input_message( + data: Any, capture_content: bool = False +) -> list[InputMessage]: """Build ``InputMessage`` s from a workflow/agent input mapping. When ``data['messages']`` is present, every LangChain ``BaseMessage`` in it @@ -397,7 +379,10 @@ def make_input_message(data: Any) -> list[InputMessage]: messages, Iterable ): return [] - return to_input_messages(cast(Iterable[BaseMessage], messages)) + return to_input_messages( + cast(Iterable[BaseMessage], messages), + capture_content, + ) # Fallback: serialize non-message state fields as input. # Common in LangGraph where nodes use structured state fields # (e.g., user_query) rather than a message list. @@ -414,7 +399,9 @@ def make_input_message(data: Any) -> list[InputMessage]: return [] -def make_output_message(data: Any) -> list[OutputMessage]: +def make_output_message( + data: Any, capture_content: bool = False +) -> list[OutputMessage]: """Build ``OutputMessage`` s from a workflow/agent output mapping. Only ``AIMessage`` entries become outputs. ``finish_reason`` is left @@ -432,17 +419,22 @@ def make_output_message(data: Any) -> list[OutputMessage]: or not isinstance(messages, Iterable) ): return [] - return to_output_messages(cast(Iterable[BaseMessage], messages)) + return to_output_messages( + cast(Iterable[BaseMessage], messages), + capture_content=capture_content, + ) -def make_last_output_message(data: Any) -> list[OutputMessage]: +def make_last_output_message( + data: Any, capture_content: bool = False +) -> list[OutputMessage]: """Extract only the last AI message as the output. For Workflow and AgentInvocation spans, the final AI message best represents the actual output. Intermediate AI messages (e.g., tool-call decisions) are already captured in child LLM invocation spans. """ - all_messages = make_output_message(data) + all_messages = make_output_message(data, capture_content) if all_messages: return [all_messages[-1]] return [] diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/multimodal.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/multimodal.py new file mode 100644 index 000000000..6008ac0e7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/multimodal.py @@ -0,0 +1,218 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenarios: langchain multimodal chat. + +Exercises an inline base64 image content block through both ChatOpenAI (an +``image_url`` data URI) and ChatAnthropic (an ``image`` block with a +``source.base64`` payload), asserting the bytes round-trip onto the input +message as an image ``Blob`` part (``type == "blob"``, +``modality == "image"``). +""" + +from __future__ import annotations + +import json +import os +from typing import Any +from unittest import mock + +from langchain_anthropic import ChatAnthropic +from langchain_core.messages import HumanMessage +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.genai.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import ( + ExpectedViolation, + Scenario, +) +from opentelemetry.test_util_genai.instrumentor import instrument + +# A tiny valid PNG, base64-encoded. Pinned to the recorded cassettes' requests. +_REAL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAIAAADYYG7QAAAARklEQVR42u3X" + "QQ0AIAwAsSnZG4lInJxJwMRICGlyAvq9yF1PFUBAQEBAQBdAXWskICAgICAg" + "ICAgICAgIOcKBAQEBPQd6ACUHHNEU5qggAAAAABJRU5ErkJggg==" +) + + +class OpenAIMultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + # langchain can't populate server.address on chat spans. + expected_violations = ( + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("OPENAI_API_KEY") + else {"OPENAI_API_KEY": "test_openai_api_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + llm = ChatOpenAI( + model="gpt-4o", + temperature=0.1, + max_tokens=100, + ) + messages = [ + HumanMessage( + content=[ + { + "type": "text", + "text": "What is in this image?", + }, + { + "type": "image_url", + "image_url": { + "url": ( + "data:image/png;base64," + f"{_REAL_PNG_B64}" + ) + }, + }, + ] + ), + ] + with vcr.use_cassette( + "test_chat_openai_multimodal_image_llm_call.yaml" + ): + llm.invoke(messages) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + _assert_input_image_blob(report) + + +class AnthropicMultimodalScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + # langchain can't populate server.address on chat spans. + expected_violations = ( + ExpectedViolation( + advice_id="genai_expected_attribute_missing", + message_substring="server.address", + ), + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("ANTHROPIC_API_KEY") + else {"ANTHROPIC_API_KEY": "test_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + llm = ChatAnthropic( + model="claude-sonnet-4-5", + temperature=0.1, + max_tokens=1024, + ) + messages = [ + HumanMessage( + content=[ + { + "type": "text", + "text": "What is in this image?", + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + }, + ] + ), + ] + with vcr.use_cassette( + "test_chat_anthropic_multimodal_image_llm_call.yaml" + ): + llm.invoke(messages) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + _assert_input_image_blob(report) + + +def _assert_input_image_blob(report: LiveCheckReport) -> None: + # Lib-specific: weaver validates each part's *shape*, but not that an + # inline image actually round-tripped. Assert the base64 image landed + # on an input message as an image blob part. + chat_spans = [ + entry["span"] + for entry in report["samples"] + if "span" in entry + and _attr(entry["span"], "gen_ai.operation.name") == "chat" + ] + assert chat_spans, "no chat span emitted" + + input_parts = { + (t, m) + for span in chat_spans + for t, m in _part_fields(_attr(span, "gen_ai.input.messages")) + } + assert ("blob", "image") in input_parts, ( + f"expected an image blob part on an input message, saw {input_parts}" + ) + + +def _attr(span: dict[str, Any], name: str) -> Any: + for attr in span["attributes"]: + if attr["name"] == name: + return attr["value"] + return None + + +def _part_fields(messages_json: str | None) -> list[tuple[str, str | None]]: + # gen_ai.{input,output}.messages is a JSON string of + # [{"role": ..., "parts": [{"type": ..., "modality": ...}]}]. Keep + # modality so image/audio/video are distinguishable on blob parts. + messages = json.loads(messages_json) if messages_json else [] + return [ + (part["type"], part.get("modality")) + for message in messages + for part in message["parts"] + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index ebccfe682..f2945e7c4 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -21,8 +21,6 @@ OpenTelemetryLangChainCallbackHandler, ) from opentelemetry.instrumentation.genai.langchain.utils import ( - _decode_base64, - _image_from_url, _legacy_function_call_request, _media_part, extract_token_details, @@ -47,6 +45,7 @@ ToolCallRequest, Uri, ) +from opentelemetry.util.genai.utils import decode_base64 # --------------------------------------------------------------------------- # Helpers @@ -1770,3 +1769,228 @@ def test_empty_header_value_ignored(self): handler.on_llm_end(response=response, run_id=run_id) assert llm_inv.response_model_name == "gpt-4o" + + +# --------------------------------------------------------------------------- +# utils._media_part - LangChain multimodal image block parsing +# --------------------------------------------------------------------------- + +_REAL_PNG_BYTES = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00" + b"\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc" + b"\xf8\xcf\xc0\xf0\x1f\x00\x05\x05\x02\x00\xa1\r\xf7\xdf\x00\x00\x00" + b"\x00IEND\xaeB`\x82" +) +_REAL_PNG_B64 = base64.b64encode(_REAL_PNG_BYTES).decode("ascii") + + +def test_media_part_openai_image_url_dict(): + item = { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/a.png" + + +def test_media_part_openai_image_url_string(): + item = {"type": "image_url", "image_url": "https://example.com/b.png"} + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/b.png" + + +def test_media_part_anthropic_base64_source_returns_blob(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "R0lGODlh", + }, + } + part = _media_part(item, capture_content=True) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == b"GIF89a" + + +def test_media_part_anthropic_base64_source_without_media_type(): + item = { + "type": "image", + "source": {"type": "base64", "data": "QUJD"}, + } + part = _media_part(item, capture_content=True) + assert isinstance(part, Blob) + assert part.mime_type is None + assert part.content == b"ABC" + + +def test_media_part_base64_source_skips_decode_when_content_disabled(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + } + with mock.patch( + "opentelemetry.instrumentation.genai.langchain.utils.decode_base64", + wraps=decode_base64, + ) as mock_decode: + part = _media_part(item, capture_content=False) + # The content-capture flag is passed into ``decode_base64`` itself, so the + # call still happens but gates internally: it returns ``None`` (no bytes + # decoded) and thus no media part is emitted. + mock_decode.assert_called_once_with(_REAL_PNG_B64, False) + assert part is None + + +def test_media_part_base64_source_decodes_when_content_enabled(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + } + with mock.patch( + "opentelemetry.instrumentation.genai.langchain.utils.decode_base64", + wraps=decode_base64, + ) as mock_decode: + part = _media_part(item, capture_content=True) + mock_decode.assert_called_once() + assert isinstance(part, Blob) + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_anthropic_url_source_returns_uri(): + item = { + "type": "image", + "source": {"type": "url", "url": "https://example.com/c.png"}, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/c.png" + + +def test_media_part_unrecognized_returns_none(): + assert _media_part({"type": "text", "text": "hi"}) is None + assert _media_part({"type": "image_url", "image_url": {}}) is None + assert ( + _media_part({"type": "image_url", "image_url": {"url": 123}}) is None + ) + assert _media_part({"type": "image", "source": "nope"}) is None + assert ( + _media_part({"type": "image", "source": {"type": "base64", "data": 5}}) + is None + ) + assert ( + _media_part({"type": "image", "source": {"type": "url", "url": ""}}) + is None + ) + + +def test_media_part_malformed_base64_returns_none(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "not!valid!base64!", + }, + } + assert _media_part(item, capture_content=True) is None + + +def test_media_part_openai_real_png_data_uri_returns_blob(): + item = { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + } + part = _media_part(item, capture_content=True) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_anthropic_real_png_source_returns_blob(): + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": _REAL_PNG_B64, + }, + } + part = _media_part(item, capture_content=True) + assert isinstance(part, Blob) + assert part.mime_type == "image/png" + assert part.content == _REAL_PNG_BYTES + + +def test_media_part_anthropic_real_png_corrupted_base64_returns_none(): + corrupted = _REAL_PNG_B64[:10] + "@@@@" + _REAL_PNG_B64[14:] + item = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": corrupted, + }, + } + assert _media_part(item, capture_content=True) is None + + +def test_media_part_real_png_url_source_returns_uri(): + item = { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/real-image.png", + }, + } + part = _media_part(item) + assert isinstance(part, Uri) + assert part.uri == "https://example.com/real-image.png" + + +def test_to_input_messages_extracts_image_part(): + image_url = "data:image/jpeg;base64,QUJD" + content = [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": image_url}}, + ] + messages = to_input_messages( + [HumanMessage(content=content)], capture_content=True + ) + assert len(messages) == 1 + parts = messages[0].parts + + assert any(isinstance(p, Blob) for p in parts) + blob = next(p for p in parts if isinstance(p, Blob)) + assert blob.mime_type == "image/jpeg" + assert blob.content == b"ABC" + + +def test_to_input_messages_skips_image_decode_when_content_disabled(): + image_url = f"data:image/png;base64,{_REAL_PNG_B64}" + content = [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": image_url}}, + ] + with mock.patch( + "opentelemetry.instrumentation.genai.langchain.utils.decode_base64", + wraps=decode_base64, + ) as mock_decode: + messages = to_input_messages( + [HumanMessage(content=content)], capture_content=False + ) + mock_decode.assert_not_called() + assert len(messages) == 1 + parts = messages[0].parts + assert any(isinstance(p, Text) for p in parts) + assert not any(isinstance(p, Blob) for p in parts) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py index 569392419..af97ede95 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_conformance.py @@ -22,6 +22,10 @@ from .conformance.agent import AgentScenario from .conformance.inference import InferenceScenario +from .conformance.multimodal import ( + AnthropicMultimodalScenario, + OpenAIMultimodalScenario, +) from .conformance.retrieval import RetrievalScenario from .conformance.tool_calling import ToolCallingScenario from .conformance.workflow import WorkflowScenario @@ -31,6 +35,8 @@ "scenario", [ InferenceScenario(), + OpenAIMultimodalScenario(), + AnthropicMultimodalScenario(), AgentScenario(), ToolCallingScenario(), WorkflowScenario(), diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index a78b46b7b..00f798847 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -4,7 +4,7 @@ import json import logging import os -from base64 import b64encode +from base64 import b64decode, b64encode from functools import partial from typing import Any @@ -12,7 +12,12 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, OTEL_INSTRUMENTATION_GENAI_EMIT_EVENT, ) -from opentelemetry.util.genai.types import ContentCapturingMode +from opentelemetry.util.genai.types import ( + Blob, + ContentCapturingMode, + MessagePart, + Uri, +) logger = logging.getLogger(__name__) @@ -36,6 +41,43 @@ def get_content_capturing_mode() -> ContentCapturingMode: return ContentCapturingMode.NO_CONTENT +def decode_base64(data: str, capture_content: bool = False) -> bytes | None: + if not capture_content: + return + try: + return b64decode("".join(data.split()), validate=True) + except Exception: # pylint: disable=broad-exception-caught + return None + + +def image_from_url( + url: str, *, modality: str = "image", capture_content: bool = False +) -> MessagePart | None: + """Return a media part for an image ``url``. + + A ``data:;base64,`` URL is decoded into a :class:`Blob`; + a ``data:`` URL without base64 encoding keeps its raw payload bytes; any + other URL becomes a :class:`Uri`. Shared by instrumentations that parse + provider image blocks. + """ + if url.startswith("data:"): + header, _, payload = url[len("data:") :].partition(",") + mime_type = header.split(";", 1)[0] or None + if ";base64" in header: + decoded = decode_base64(payload, capture_content) + if decoded is None: + return None + content = decoded + else: + content = payload.encode("utf-8") + return Blob( + mime_type=mime_type, + modality=modality, + content=content, + ) + return Uri(mime_type=None, modality=modality, uri=url) + + def is_experimental_mode() -> bool: """ Kept for backwards compatibility. The utils in this library only support the experimental mode sem convs now. diff --git a/util/opentelemetry-util-genai/tests/test_utils.py b/util/opentelemetry-util-genai/tests/test_utils.py index 127e9abae..5cbe2b621 100644 --- a/util/opentelemetry-util-genai/tests/test_utils.py +++ b/util/opentelemetry-util-genai/tests/test_utils.py @@ -1,6 +1,7 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +import base64 import json import os import unittest @@ -34,14 +35,18 @@ get_telemetry_handler, ) from opentelemetry.util.genai.types import ( + Blob, ContentCapturingMode, InputMessage, MessagePart, OutputMessage, Text, + Uri, ) from opentelemetry.util.genai.utils import ( + decode_base64, get_content_capturing_mode, + image_from_url, should_capture_content_on_spans, should_emit_event, ) @@ -956,3 +961,144 @@ def test_inference_finish_does_not_duplicate_start_attributes(self): class AnyNonNone: def __eq__(self, other): return other is not None + + +_REAL_PNG_BYTES = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00" + b"\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc" + b"\xf8\xcf\xc0\xf0\x1f\x00\x05\x05\x02\x00\xa1\r\xf7\xdf\x00\x00\x00" + b"\x00IEND\xaeB`\x82" +) +_REAL_PNG_B64 = base64.b64encode(_REAL_PNG_BYTES).decode("ascii") + + +class TestMediaHelpers(unittest.TestCase): + """Tests for the shared ``decode_base64`` / ``image_from_url`` helpers. + + ``decode_base64`` and ``image_from_url`` only decode inline base64 + payloads when their ``capture_content`` flag (resolved once up the stack) + is ``True``; with the flag off they short-circuit and return ``None`` + without decoding. The flag is passed explicitly and never read from the + environment. + """ + + # -- image_from_url -------------------------------------------------- + + def test_image_from_url_data_uri_returns_blob(self): + part = image_from_url( + "data:image/jpeg;base64,QUJD", capture_content=True + ) + self.assertIsInstance(part, Blob) + self.assertEqual(part.mime_type, "image/jpeg") + self.assertEqual(part.modality, "image") + self.assertEqual(part.content, b"ABC") + + def test_image_from_url_data_uri_skips_decode_without_capture_content( + self, + ): + self.assertIsNone(image_from_url("data:image/jpeg;base64,QUJD")) + + def test_image_from_url_http_returns_uri(self): + part = image_from_url("https://example.com/cat.png") + self.assertIsInstance(part, Uri) + self.assertEqual(part.uri, "https://example.com/cat.png") + self.assertEqual(part.modality, "image") + + def test_image_from_url_data_uri_without_base64_keeps_text_bytes(self): + part = image_from_url("data:text/plain,hello") + self.assertIsInstance(part, Blob) + self.assertEqual(part.mime_type, "text/plain") + self.assertEqual(part.content, b"hello") + + def test_image_from_url_data_uri_no_mime_type(self): + part = image_from_url("data:;base64,QUJD", capture_content=True) + self.assertIsInstance(part, Blob) + self.assertIsNone(part.mime_type) + self.assertEqual(part.content, b"ABC") + + def test_image_from_url_data_uri_malformed_base64_returns_none(self): + part = image_from_url( + "data:image/png;base64,not!valid!base64!", capture_content=True + ) + self.assertIsNone(part) + + def test_image_from_url_honours_modality_override(self): + part = image_from_url( + "data:audio/mp3;base64,QUJD", + modality="audio", + capture_content=True, + ) + self.assertIsInstance(part, Blob) + self.assertEqual(part.modality, "audio") + uri_part = image_from_url( + "https://example.com/a.mp3", modality="audio" + ) + self.assertIsInstance(uri_part, Uri) + self.assertEqual(uri_part.modality, "audio") + + # -- decode_base64 --------------------------------------------------- + + def test_decode_base64_valid_returns_bytes(self): + self.assertEqual(decode_base64("QUJD", capture_content=True), b"ABC") + + def test_decode_base64_valid_with_padding(self): + self.assertEqual( + decode_base64("R0lGODlh", capture_content=True), b"GIF89a" + ) + + def test_decode_base64_strips_whitespace_and_newlines(self): + self.assertEqual(decode_base64("QU\nJD", capture_content=True), b"ABC") + self.assertEqual( + decode_base64(" QUJD ", capture_content=True), b"ABC" + ) + self.assertEqual(decode_base64("QU JD", capture_content=True), b"ABC") + + def test_decode_base64_malformed_returns_none(self): + self.assertIsNone( + decode_base64("not!valid!base64!", capture_content=True) + ) + self.assertIsNone(decode_base64("@@@@", capture_content=True)) + self.assertIsNone(decode_base64("****", capture_content=True)) + + def test_decode_base64_wrong_padding_returns_none(self): + # Correct base64 alphabet but invalid length/padding. + self.assertIsNone(decode_base64("QUJ", capture_content=True)) + self.assertIsNone(decode_base64("QQ", capture_content=True)) + + def test_decode_base64_empty_returns_empty_bytes(self): + self.assertEqual(decode_base64("", capture_content=True), b"") + + def test_decode_base64_real_image_round_trips(self): + self.assertEqual( + decode_base64(_REAL_PNG_B64, capture_content=True), _REAL_PNG_BYTES + ) + + def test_decode_base64_without_capture_content_returns_none(self): + self.assertIsNone(decode_base64("QUJD")) + self.assertIsNone(decode_base64(_REAL_PNG_B64, capture_content=False)) + + @patch.dict( + os.environ, + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT"}, + ) + def test_decode_base64_ignores_env_and_uses_flag(self): + self.assertEqual(decode_base64("QUJD", capture_content=True), b"ABC") + self.assertEqual( + decode_base64(_REAL_PNG_B64, capture_content=True), _REAL_PNG_BYTES + ) + self.assertIsNone(decode_base64("QUJD", capture_content=False)) + + def test_image_from_url_real_png_data_uri_returns_blob(self): + part = image_from_url( + f"data:image/png;base64,{_REAL_PNG_B64}", capture_content=True + ) + self.assertIsInstance(part, Blob) + self.assertEqual(part.mime_type, "image/png") + self.assertEqual(part.content, _REAL_PNG_BYTES) + + def test_image_from_url_real_png_truncated_base64_returns_none(self): + corrupted = _REAL_PNG_B64[:-4] + "!!!!" + part = image_from_url( + f"data:image/png;base64,{corrupted}", capture_content=True + ) + self.assertIsNone(part) From a779861362af0c8f22fed1aac2aff7df7ca9278a Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 10 Aug 2026 16:40:22 -0700 Subject: [PATCH 09/14] Fix tests and docstring --- .../tests/test_llm_call.py | 124 +++++++++++------- .../src/opentelemetry/util/genai/utils.py | 9 +- 2 files changed, 82 insertions(+), 51 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 07fa5f44f..e9957595e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -15,6 +15,9 @@ from langchain_core.tools import tool from openai import AuthenticationError +from opentelemetry.instrumentation.genai.langchain import ( + LangChainInstrumentor, +) from opentelemetry.instrumentation.genai.langchain.utils import ( to_input_messages, ) @@ -206,17 +209,15 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( def test_chat_openai_multimodal_image_llm_call( span_exporter, - start_instrumentation, + tracer_provider, + meter_provider, + logger_provider, chat_openai_vision, monkeypatch, vcr, ): """End-to-end: an OpenAI ``image_url`` content block is captured as an image ``Blob`` part in ``gen_ai.input.messages``.""" - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" - ) - messages = [ HumanMessage( content=[ @@ -237,28 +238,45 @@ def test_chat_openai_multimodal_image_llm_call( "langchain-openai < 1.0 sends a different request body " "(explicit n/temperature); only the modern cassette is recorded" ) - with vcr.use_cassette("test_chat_openai_multimodal_image_llm_call.yaml"): - chat_openai_vision.invoke(messages) - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - span = spans[0] - - assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( - "gpt-4o" + # The content-capture flag is cached when the handler is built at + # ``instrument()`` time, so set it before instrumenting (not after). + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" ) - - input_message = span.attributes.get( - gen_ai_attributes.GEN_AI_INPUT_MESSAGES + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, ) - assert input_message is not None - assert '"role":"user"' in input_message - assert '"type":"text"' in input_message - assert '"content":"What is in this image?"' in input_message - assert '"type":"blob"' in input_message - assert '"modality":"image"' in input_message - assert '"mime_type":"image/png"' in input_message - assert _REAL_PNG_B64 in input_message + try: + with vcr.use_cassette( + "test_chat_openai_multimodal_image_llm_call.yaml" + ): + chat_openai_vision.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + assert span.attributes.get( + gen_ai_attributes.GEN_AI_REQUEST_MODEL + ) == ("gpt-4o") + + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + finally: + instrumentor.uninstrument() # span_exporter, start_instrumentation, us_amazon_nova_lite_v1_0 are coming from fixtures defined in conftest.py @@ -440,16 +458,14 @@ def test_chat_openai_legacy_function_call( @pytest.mark.vcr() def test_chat_anthropic_multimodal_image_llm_call( span_exporter, - start_instrumentation, + tracer_provider, + meter_provider, + logger_provider, chat_anthropic_claude_sonnet, monkeypatch, ): """End-to-end: an Anthropic ``image`` content block is captured as an image ``Blob`` part in ``gen_ai.input.messages``.""" - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" - ) - messages = [ HumanMessage( content=[ @@ -466,27 +482,41 @@ def test_chat_anthropic_multimodal_image_llm_call( ), ] - chat_anthropic_claude_sonnet.invoke(messages) + # The content-capture flag is cached when the handler is built at + # ``instrument()`` time, so set it before instrumenting (not after). + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "SPAN_ONLY" + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + try: + chat_anthropic_claude_sonnet.invoke(messages) - spans = span_exporter.get_finished_spans() - assert len(spans) == 1 - span = spans[0] + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] - assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( - "claude-sonnet-4-5" - ) + assert span.attributes.get( + gen_ai_attributes.GEN_AI_REQUEST_MODEL + ) == ("claude-sonnet-4-5") - input_message = span.attributes.get( - gen_ai_attributes.GEN_AI_INPUT_MESSAGES - ) - assert input_message is not None - assert '"role":"user"' in input_message - assert '"type":"text"' in input_message - assert '"content":"What is in this image?"' in input_message - assert '"type":"blob"' in input_message - assert '"modality":"image"' in input_message - assert '"mime_type":"image/png"' in input_message - assert _REAL_PNG_B64 in input_message + input_message = span.attributes.get( + gen_ai_attributes.GEN_AI_INPUT_MESSAGES + ) + assert input_message is not None + assert '"role":"user"' in input_message + assert '"type":"text"' in input_message + assert '"content":"What is in this image?"' in input_message + assert '"type":"blob"' in input_message + assert '"modality":"image"' in input_message + assert '"mime_type":"image/png"' in input_message + assert _REAL_PNG_B64 in input_message + finally: + instrumentor.uninstrument() # span_exporter, start_instrumentation, gemini are coming from fixtures defined in conftest.py diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index 00f798847..c381676b3 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -55,10 +55,11 @@ def image_from_url( ) -> MessagePart | None: """Return a media part for an image ``url``. - A ``data:;base64,`` URL is decoded into a :class:`Blob`; - a ``data:`` URL without base64 encoding keeps its raw payload bytes; any - other URL becomes a :class:`Uri`. Shared by instrumentations that parse - provider image blocks. + A ``data:;base64,`` URL is decoded into a + :class:`~opentelemetry.util.genai.types.Blob`; a ``data:`` URL without + base64 encoding keeps its raw payload bytes; any other URL becomes a + :class:`~opentelemetry.util.genai.types.Uri`. Shared by instrumentations + that parse provider image blocks. """ if url.startswith("data:"): header, _, payload = url[len("data:") :].partition(",") From 855816660d863b9babd14f057fd0cdbe9650c182 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Mon, 10 Aug 2026 16:49:25 -0700 Subject: [PATCH 10/14] Fix format --- .../tests/test_llm_call.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index e9957595e..ca109250c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -260,9 +260,9 @@ def test_chat_openai_multimodal_image_llm_call( assert len(spans) == 1 span = spans[0] - assert span.attributes.get( - gen_ai_attributes.GEN_AI_REQUEST_MODEL - ) == ("gpt-4o") + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "gpt-4o" + ) input_message = span.attributes.get( gen_ai_attributes.GEN_AI_INPUT_MESSAGES @@ -500,9 +500,9 @@ def test_chat_anthropic_multimodal_image_llm_call( assert len(spans) == 1 span = spans[0] - assert span.attributes.get( - gen_ai_attributes.GEN_AI_REQUEST_MODEL - ) == ("claude-sonnet-4-5") + assert span.attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == ( + "claude-sonnet-4-5" + ) input_message = span.attributes.get( gen_ai_attributes.GEN_AI_INPUT_MESSAGES From a7d14770f43c63481f002622a883e4d635fe8787 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 18 Aug 2026 09:22:43 -0700 Subject: [PATCH 11/14] Address feedback --- .../genai/langchain/callback_handler.py | 22 ++-- .../instrumentation/genai/langchain/utils.py | 79 ++++++-------- .../tests/test_callback_handler.py | 103 +++++++++--------- .../src/opentelemetry/util/genai/utils.py | 16 +-- .../tests/test_utils.py | 84 +++++--------- 5 files changed, 131 insertions(+), 173 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 12e7e35c1..68e2ca598 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -40,6 +40,7 @@ WorkflowInvocation, ) from opentelemetry.util.genai.types import ( + InputMessage, MessagePart, OutputMessage, Text, @@ -82,9 +83,8 @@ def on_chain_start( workflow = self._telemetry_handler.workflow( name=workflow_name_override or workflow_name ) - workflow.input_messages = make_input_message( - inputs, capture_content - ) + if capture_content: + workflow.input_messages = make_input_message(inputs) self._invocation_manager.add_invocation_state( run_id, parent_run_id, workflow ) @@ -109,9 +109,8 @@ def on_chain_start( agent = self._telemetry_handler.invoke_local_agent( agent_name=suggested_agent_name, ) - agent.input_messages = make_input_message( - inputs, capture_content - ) + if capture_content: + agent.input_messages = make_input_message(inputs) if metadata: agent.agent_id = metadata.get("agent_id") @@ -166,10 +165,8 @@ def on_chain_end( self._invocation_manager.delete_invocation_state(run_id) return - capture_content = self._telemetry_handler.should_capture_content() - invocation.output_messages = make_last_output_message( - outputs, capture_content - ) + if self._telemetry_handler.should_capture_content(): + invocation.output_messages = make_last_output_message(outputs) invocation.stop() @@ -277,8 +274,9 @@ def on_chat_model_start( # :func:`to_input_messages` produce spec-conformant ``InputMessage`` s # with proper roles, tool-call requests, tool results, and reasoning. flattened: list[BaseMessage] = [msg for sub in messages for msg in sub] - capture_content = self._telemetry_handler.should_capture_content() - input_messages = to_input_messages(flattened, capture_content) + input_messages: list[InputMessage] = [] + if self._telemetry_handler.should_capture_content(): + input_messages = to_input_messages(flattened) llm_invocation = self._telemetry_handler.inference( provider, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index e575b1c9a..5a9ceaf6e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -77,9 +77,7 @@ def _normalize_role(message: BaseMessage) -> str: return _ROLE_MAP.get(message.type, message.type) -def _media_part( - item: dict[str, Any], capture_content: bool = False -) -> MessagePart | None: +def _media_part(item: dict[str, Any]) -> MessagePart | None: """Convert a LangChain multimodal image content block into a media part. Handles the two shapes LangChain chat models accept: @@ -103,7 +101,7 @@ def _media_part( url = raw_url if isinstance(raw_url, str) else None if not url: return None - return image_from_url(url, capture_content=capture_content) + return image_from_url(url) if block_type == "image": source = item.get("source") if not isinstance(source, dict): @@ -114,7 +112,7 @@ def _media_part( data = source_dict.get("data") if not isinstance(data, str): return None - decoded = decode_base64(data, capture_content) + decoded = decode_base64(data) if decoded is None: return None media_type = source_dict.get("media_type") @@ -128,15 +126,12 @@ def _media_part( if source_type == "url": source_url = source_dict.get("url") if isinstance(source_url, str) and source_url: - return image_from_url( - source_url, capture_content=capture_content - ) + return image_from_url(source_url) return None def _content_to_parts( content: str | list[str | dict[str, Any]], - capture_content: bool = False, ) -> list[MessagePart]: """Convert a LangChain message ``content`` payload into ``MessagePart`` s. @@ -170,7 +165,7 @@ def _content_to_parts( if isinstance(reasoning_value, str) and reasoning_value: parts.append(Reasoning(content=reasoning_value)) elif block_type in ("image_url", "image"): - media = _media_part(item, capture_content) + media = _media_part(item) if media is not None: parts.append(media) return parts @@ -203,18 +198,14 @@ def _legacy_function_call_request( return ToolCallRequest(arguments=arguments, name=name, id=None) -def _ai_message_parts( - message: AIMessage, capture_content: bool = False -) -> list[MessagePart]: +def _ai_message_parts(message: AIMessage) -> list[MessagePart]: """Build :class:`MessagePart` s for an :class:`AIMessage`. Includes any text/reasoning content followed by a :class:`ToolCallRequest` for each entry in ``message.tool_calls``, plus a legacy ``additional_kwargs['function_call']`` when present. """ - parts: list[MessagePart] = _content_to_parts( - message.content, capture_content - ) + parts: list[MessagePart] = _content_to_parts(message.content) for call in message.tool_calls: name = call["name"] if not name: @@ -244,21 +235,22 @@ def _tool_message_parts(message: ToolMessage) -> list[MessagePart]: ] -def _message_parts( - message: BaseMessage, capture_content: bool = False -) -> list[MessagePart]: +def _message_parts(message: BaseMessage) -> list[MessagePart]: if isinstance(message, ToolMessage): return _tool_message_parts(message) if isinstance(message, AIMessage): - return _ai_message_parts(message, capture_content) - return _content_to_parts(message.content, capture_content) + return _ai_message_parts(message) + return _content_to_parts(message.content) def to_input_messages( messages: Iterable[Any], - capture_content: bool = False, ) -> list[InputMessage]: - """Convert LangChain messages into spec-conformant ``InputMessage`` s.""" + """Convert LangChain messages into spec-conformant ``InputMessage`` s. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). + """ try: normalized_messages: Iterable[BaseMessage] = convert_to_messages( list(messages) @@ -269,7 +261,7 @@ def to_input_messages( ] result: list[InputMessage] = [] for message in normalized_messages: - parts = _message_parts(message, capture_content) + parts = _message_parts(message) if not parts: continue result.append(InputMessage(role=_normalize_role(message), parts=parts)) @@ -280,7 +272,6 @@ def to_output_messages( messages: Iterable[BaseMessage], *, finish_reason: str = "", - capture_content: bool = False, ) -> list[OutputMessage]: """Convert LangChain ``AIMessage`` instances into ``OutputMessage`` s. @@ -288,12 +279,15 @@ def to_output_messages( as ``gen_ai.output.messages``. Tool execution results belong on the *input* side of the next inference call, not the output side of the previous one. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ result: list[OutputMessage] = [] for message in messages: if not isinstance(message, AIMessage): continue - parts = _ai_message_parts(message, capture_content) + parts = _ai_message_parts(message) if not parts: continue result.append( @@ -355,9 +349,7 @@ def prepare_tool_definitions(tools: list[Any]) -> list[ToolDefinition] | None: return definitions or None -def make_input_message( - data: Any, capture_content: bool = False -) -> list[InputMessage]: +def make_input_message(data: Any) -> list[InputMessage]: """Build ``InputMessage`` s from a workflow/agent input mapping. When ``data['messages']`` is present, every LangChain ``BaseMessage`` in it @@ -369,6 +361,9 @@ def make_input_message( When no ``messages`` key exists (common in LangGraph state dicts), the remaining state fields are serialized as JSON and emitted as a single user-role :class:`Text` part. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ if not isinstance(data, dict): return [] @@ -379,10 +374,7 @@ def make_input_message( messages, Iterable ): return [] - return to_input_messages( - cast(Iterable[BaseMessage], messages), - capture_content, - ) + return to_input_messages(cast(Iterable[BaseMessage], messages)) # Fallback: serialize non-message state fields as input. # Common in LangGraph where nodes use structured state fields # (e.g., user_query) rather than a message list. @@ -399,15 +391,16 @@ def make_input_message( return [] -def make_output_message( - data: Any, capture_content: bool = False -) -> list[OutputMessage]: +def make_output_message(data: Any) -> list[OutputMessage]: """Build ``OutputMessage`` s from a workflow/agent output mapping. Only ``AIMessage`` entries become outputs. ``finish_reason`` is left empty: the underlying per-LLM-call finish reasons are recorded on child inference spans, and util-genai filters empty values out of ``gen_ai.response.finish_reasons``. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ if not isinstance(data, dict): return [] @@ -419,22 +412,20 @@ def make_output_message( or not isinstance(messages, Iterable) ): return [] - return to_output_messages( - cast(Iterable[BaseMessage], messages), - capture_content=capture_content, - ) + return to_output_messages(cast(Iterable[BaseMessage], messages)) -def make_last_output_message( - data: Any, capture_content: bool = False -) -> list[OutputMessage]: +def make_last_output_message(data: Any) -> list[OutputMessage]: """Extract only the last AI message as the output. For Workflow and AgentInvocation spans, the final AI message best represents the actual output. Intermediate AI messages (e.g., tool-call decisions) are already captured in child LLM invocation spans. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ - all_messages = make_output_message(data, capture_content) + all_messages = make_output_message(data) if all_messages: return [all_messages[-1]] return [] diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py index f2945e7c4..a8748bc59 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_callback_handler.py @@ -45,7 +45,6 @@ ToolCallRequest, Uri, ) -from opentelemetry.util.genai.utils import decode_base64 # --------------------------------------------------------------------------- # Helpers @@ -1810,7 +1809,7 @@ def test_media_part_anthropic_base64_source_returns_blob(): "data": "R0lGODlh", }, } - part = _media_part(item, capture_content=True) + part = _media_part(item) assert isinstance(part, Blob) assert part.mime_type == "image/png" assert part.content == b"GIF89a" @@ -1821,34 +1820,13 @@ def test_media_part_anthropic_base64_source_without_media_type(): "type": "image", "source": {"type": "base64", "data": "QUJD"}, } - part = _media_part(item, capture_content=True) + part = _media_part(item) assert isinstance(part, Blob) assert part.mime_type is None assert part.content == b"ABC" -def test_media_part_base64_source_skips_decode_when_content_disabled(): - item = { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": _REAL_PNG_B64, - }, - } - with mock.patch( - "opentelemetry.instrumentation.genai.langchain.utils.decode_base64", - wraps=decode_base64, - ) as mock_decode: - part = _media_part(item, capture_content=False) - # The content-capture flag is passed into ``decode_base64`` itself, so the - # call still happens but gates internally: it returns ``None`` (no bytes - # decoded) and thus no media part is emitted. - mock_decode.assert_called_once_with(_REAL_PNG_B64, False) - assert part is None - - -def test_media_part_base64_source_decodes_when_content_enabled(): +def test_media_part_base64_source_decodes_real_png(): item = { "type": "image", "source": { @@ -1857,12 +1835,7 @@ def test_media_part_base64_source_decodes_when_content_enabled(): "data": _REAL_PNG_B64, }, } - with mock.patch( - "opentelemetry.instrumentation.genai.langchain.utils.decode_base64", - wraps=decode_base64, - ) as mock_decode: - part = _media_part(item, capture_content=True) - mock_decode.assert_called_once() + part = _media_part(item) assert isinstance(part, Blob) assert part.content == _REAL_PNG_BYTES @@ -1903,7 +1876,7 @@ def test_media_part_malformed_base64_returns_none(): "data": "not!valid!base64!", }, } - assert _media_part(item, capture_content=True) is None + assert _media_part(item) is None def test_media_part_openai_real_png_data_uri_returns_blob(): @@ -1911,7 +1884,7 @@ def test_media_part_openai_real_png_data_uri_returns_blob(): "type": "image_url", "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, } - part = _media_part(item, capture_content=True) + part = _media_part(item) assert isinstance(part, Blob) assert part.mime_type == "image/png" assert part.content == _REAL_PNG_BYTES @@ -1926,7 +1899,7 @@ def test_media_part_anthropic_real_png_source_returns_blob(): "data": _REAL_PNG_B64, }, } - part = _media_part(item, capture_content=True) + part = _media_part(item) assert isinstance(part, Blob) assert part.mime_type == "image/png" assert part.content == _REAL_PNG_BYTES @@ -1942,7 +1915,7 @@ def test_media_part_anthropic_real_png_corrupted_base64_returns_none(): "data": corrupted, }, } - assert _media_part(item, capture_content=True) is None + assert _media_part(item) is None def test_media_part_real_png_url_source_returns_uri(): @@ -1964,9 +1937,7 @@ def test_to_input_messages_extracts_image_part(): {"type": "text", "text": "What's in this image?"}, {"type": "image_url", "image_url": {"url": image_url}}, ] - messages = to_input_messages( - [HumanMessage(content=content)], capture_content=True - ) + messages = to_input_messages([HumanMessage(content=content)]) assert len(messages) == 1 parts = messages[0].parts @@ -1976,21 +1947,49 @@ def test_to_input_messages_extracts_image_part(): assert blob.content == b"ABC" -def test_to_input_messages_skips_image_decode_when_content_disabled(): - image_url = f"data:image/png;base64,{_REAL_PNG_B64}" +def test_on_chat_model_start_skips_input_messages_when_content_disabled(): + """Gating happens in the callback handler, not inside the utils.""" + run_id = _run_id() + handler, telemetry, llm_inv = _make_handler_with_llm_invocation(run_id) + telemetry.should_capture_content.return_value = False + content = [ {"type": "text", "text": "What's in this image?"}, - {"type": "image_url", "image_url": {"url": image_url}}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + }, ] - with mock.patch( - "opentelemetry.instrumentation.genai.langchain.utils.decode_base64", - wraps=decode_base64, - ) as mock_decode: - messages = to_input_messages( - [HumanMessage(content=content)], capture_content=False - ) - mock_decode.assert_not_called() - assert len(messages) == 1 - parts = messages[0].parts + handler.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content=content)]], + run_id=_run_id(), + invocation_params={"model_name": "gpt-4o"}, + ) + + assert llm_inv.input_messages == [] + + +def test_on_chat_model_start_captures_input_messages_when_content_enabled(): + run_id = _run_id() + handler, telemetry, llm_inv = _make_handler_with_llm_invocation(run_id) + telemetry.should_capture_content.return_value = True + + content = [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_REAL_PNG_B64}"}, + }, + ] + handler.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content=content)]], + run_id=_run_id(), + invocation_params={"model_name": "gpt-4o"}, + ) + + parts = llm_inv.input_messages[0].parts assert any(isinstance(p, Text) for p in parts) - assert not any(isinstance(p, Blob) for p in parts) + blob = next(p for p in parts if isinstance(p, Blob)) + assert blob.content == _REAL_PNG_BYTES diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index c381676b3..cd3f08e74 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -41,18 +41,17 @@ def get_content_capturing_mode() -> ContentCapturingMode: return ContentCapturingMode.NO_CONTENT -def decode_base64(data: str, capture_content: bool = False) -> bytes | None: - if not capture_content: - return +def decode_base64(data: str) -> bytes | None: + """Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). + """ try: return b64decode("".join(data.split()), validate=True) except Exception: # pylint: disable=broad-exception-caught return None -def image_from_url( - url: str, *, modality: str = "image", capture_content: bool = False -) -> MessagePart | None: +def image_from_url(url: str, *, modality: str = "image") -> MessagePart | None: """Return a media part for an image ``url``. A ``data:;base64,`` URL is decoded into a @@ -60,12 +59,15 @@ def image_from_url( base64 encoding keeps its raw payload bytes; any other URL becomes a :class:`~opentelemetry.util.genai.types.Uri`. Shared by instrumentations that parse provider image blocks. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). """ if url.startswith("data:"): header, _, payload = url[len("data:") :].partition(",") mime_type = header.split(";", 1)[0] or None if ";base64" in header: - decoded = decode_base64(payload, capture_content) + decoded = decode_base64(payload) if decoded is None: return None content = decoded diff --git a/util/opentelemetry-util-genai/tests/test_utils.py b/util/opentelemetry-util-genai/tests/test_utils.py index 5cbe2b621..977dcd255 100644 --- a/util/opentelemetry-util-genai/tests/test_utils.py +++ b/util/opentelemetry-util-genai/tests/test_utils.py @@ -975,29 +975,20 @@ def __eq__(self, other): class TestMediaHelpers(unittest.TestCase): """Tests for the shared ``decode_base64`` / ``image_from_url`` helpers. - ``decode_base64`` and ``image_from_url`` only decode inline base64 - payloads when their ``capture_content`` flag (resolved once up the stack) - is ``True``; with the flag off they short-circuit and return ``None`` - without decoding. The flag is passed explicitly and never read from the - environment. + Both helpers unconditionally decode the payload they are given; callers are + expected to check whether content capture is enabled and skip calling them + when it is not. """ # -- image_from_url -------------------------------------------------- def test_image_from_url_data_uri_returns_blob(self): - part = image_from_url( - "data:image/jpeg;base64,QUJD", capture_content=True - ) + part = image_from_url("data:image/jpeg;base64,QUJD") self.assertIsInstance(part, Blob) self.assertEqual(part.mime_type, "image/jpeg") self.assertEqual(part.modality, "image") self.assertEqual(part.content, b"ABC") - def test_image_from_url_data_uri_skips_decode_without_capture_content( - self, - ): - self.assertIsNone(image_from_url("data:image/jpeg;base64,QUJD")) - def test_image_from_url_http_returns_uri(self): part = image_from_url("https://example.com/cat.png") self.assertIsInstance(part, Uri) @@ -1011,23 +1002,17 @@ def test_image_from_url_data_uri_without_base64_keeps_text_bytes(self): self.assertEqual(part.content, b"hello") def test_image_from_url_data_uri_no_mime_type(self): - part = image_from_url("data:;base64,QUJD", capture_content=True) + part = image_from_url("data:;base64,QUJD") self.assertIsInstance(part, Blob) self.assertIsNone(part.mime_type) self.assertEqual(part.content, b"ABC") def test_image_from_url_data_uri_malformed_base64_returns_none(self): - part = image_from_url( - "data:image/png;base64,not!valid!base64!", capture_content=True - ) + part = image_from_url("data:image/png;base64,not!valid!base64!") self.assertIsNone(part) def test_image_from_url_honours_modality_override(self): - part = image_from_url( - "data:audio/mp3;base64,QUJD", - modality="audio", - capture_content=True, - ) + part = image_from_url("data:audio/mp3;base64,QUJD", modality="audio") self.assertIsInstance(part, Blob) self.assertEqual(part.modality, "audio") uri_part = image_from_url( @@ -1039,66 +1024,49 @@ def test_image_from_url_honours_modality_override(self): # -- decode_base64 --------------------------------------------------- def test_decode_base64_valid_returns_bytes(self): - self.assertEqual(decode_base64("QUJD", capture_content=True), b"ABC") + self.assertEqual(decode_base64("QUJD"), b"ABC") def test_decode_base64_valid_with_padding(self): - self.assertEqual( - decode_base64("R0lGODlh", capture_content=True), b"GIF89a" - ) + self.assertEqual(decode_base64("R0lGODlh"), b"GIF89a") def test_decode_base64_strips_whitespace_and_newlines(self): - self.assertEqual(decode_base64("QU\nJD", capture_content=True), b"ABC") - self.assertEqual( - decode_base64(" QUJD ", capture_content=True), b"ABC" - ) - self.assertEqual(decode_base64("QU JD", capture_content=True), b"ABC") + self.assertEqual(decode_base64("QU\nJD"), b"ABC") + self.assertEqual(decode_base64(" QUJD "), b"ABC") + self.assertEqual(decode_base64("QU JD"), b"ABC") def test_decode_base64_malformed_returns_none(self): - self.assertIsNone( - decode_base64("not!valid!base64!", capture_content=True) - ) - self.assertIsNone(decode_base64("@@@@", capture_content=True)) - self.assertIsNone(decode_base64("****", capture_content=True)) + self.assertIsNone(decode_base64("not!valid!base64!")) + self.assertIsNone(decode_base64("@@@@")) + self.assertIsNone(decode_base64("****")) def test_decode_base64_wrong_padding_returns_none(self): # Correct base64 alphabet but invalid length/padding. - self.assertIsNone(decode_base64("QUJ", capture_content=True)) - self.assertIsNone(decode_base64("QQ", capture_content=True)) + self.assertIsNone(decode_base64("QUJ")) + self.assertIsNone(decode_base64("QQ")) def test_decode_base64_empty_returns_empty_bytes(self): - self.assertEqual(decode_base64("", capture_content=True), b"") + self.assertEqual(decode_base64(""), b"") def test_decode_base64_real_image_round_trips(self): - self.assertEqual( - decode_base64(_REAL_PNG_B64, capture_content=True), _REAL_PNG_BYTES - ) - - def test_decode_base64_without_capture_content_returns_none(self): - self.assertIsNone(decode_base64("QUJD")) - self.assertIsNone(decode_base64(_REAL_PNG_B64, capture_content=False)) + self.assertEqual(decode_base64(_REAL_PNG_B64), _REAL_PNG_BYTES) @patch.dict( os.environ, {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "NO_CONTENT"}, ) - def test_decode_base64_ignores_env_and_uses_flag(self): - self.assertEqual(decode_base64("QUJD", capture_content=True), b"ABC") - self.assertEqual( - decode_base64(_REAL_PNG_B64, capture_content=True), _REAL_PNG_BYTES - ) - self.assertIsNone(decode_base64("QUJD", capture_content=False)) + def test_decode_base64_ignores_content_capture_env(self): + # Gating is the caller's responsibility, so the helper never reads the + # content-capture environment variable itself. + self.assertEqual(decode_base64("QUJD"), b"ABC") + self.assertEqual(decode_base64(_REAL_PNG_B64), _REAL_PNG_BYTES) def test_image_from_url_real_png_data_uri_returns_blob(self): - part = image_from_url( - f"data:image/png;base64,{_REAL_PNG_B64}", capture_content=True - ) + part = image_from_url(f"data:image/png;base64,{_REAL_PNG_B64}") self.assertIsInstance(part, Blob) self.assertEqual(part.mime_type, "image/png") self.assertEqual(part.content, _REAL_PNG_BYTES) def test_image_from_url_real_png_truncated_base64_returns_none(self): corrupted = _REAL_PNG_B64[:-4] + "!!!!" - part = image_from_url( - f"data:image/png;base64,{corrupted}", capture_content=True - ) + part = image_from_url(f"data:image/png;base64,{corrupted}") self.assertIsNone(part) From b7cd8a6760f08d7cd5cf1c50aa3f22e088d0162c Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 18 Aug 2026 16:47:02 -0700 Subject: [PATCH 12/14] Retrigger CI/CD pipeline From 0447f86cc3069e9e13455b2ca3ee525927a7ba86 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 19 Aug 2026 09:22:29 -0700 Subject: [PATCH 13/14] Fix tests --- .../tests/test_llm_call.py | 67 +++++++++++-------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index ca109250c..3e079acb8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -28,6 +28,7 @@ from opentelemetry.semconv._incubating.attributes import gen_ai_attributes from opentelemetry.semconv._incubating.metrics import gen_ai_metrics from opentelemetry.semconv.attributes import error_attributes +from opentelemetry.test_util_genai.instrumentor import instrument def _openai_cassette_name(model, base: str) -> str: @@ -77,28 +78,32 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call( span_exporter, metric_reader, log_exporter, - start_instrumentation, + tracer_provider, + meter_provider, + logger_provider, chat_openai_gpt_3_5_turbo_model, - monkeypatch, capture_content, vcr, ): - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content - ) - messages = [ SystemMessage(content="You are a helpful assistant!"), HumanMessage(content="What is the capital of France?"), ] - with vcr.use_cassette( - _openai_cassette_name( - chat_openai_gpt_3_5_turbo_model, - "test_chat_openai_gpt_3_5_turbo_model_llm_call", - ) + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture=capture_content, ): - response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + with vcr.use_cassette( + _openai_cassette_name( + chat_openai_gpt_3_5_turbo_model, + "test_chat_openai_gpt_3_5_turbo_model_llm_call", + ) + ): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) assert response.content == "The capital of France is Paris." # verify spans @@ -147,33 +152,37 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error( span_exporter, metric_reader, log_exporter, - start_instrumentation, + tracer_provider, + meter_provider, + logger_provider, chat_openai_gpt_3_5_turbo_model, - monkeypatch, capture_content, vcr, ): - monkeypatch.setenv( - "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content - ) - messages = [ SystemMessage(content="You are a helpful assistant!"), HumanMessage(content="What is the capital of France?"), ] response = None - try: - with vcr.use_cassette( - _openai_cassette_name( - chat_openai_gpt_3_5_turbo_model, - "test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error", - ) - ): - response = chat_openai_gpt_3_5_turbo_model.invoke(messages) - except Exception as e: - # For this test, to get error, cassettes were recorded with no OPENAI_API_KEY, so an error is expected here. - assert isinstance(e, AuthenticationError) + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + content_capture=capture_content, + ): + try: + with vcr.use_cassette( + _openai_cassette_name( + chat_openai_gpt_3_5_turbo_model, + "test_chat_openai_gpt_3_5_turbo_model_llm_call_with_error", + ) + ): + response = chat_openai_gpt_3_5_turbo_model.invoke(messages) + except Exception as e: + # For this test, to get error, cassettes were recorded with no OPENAI_API_KEY, so an error is expected here. + assert isinstance(e, AuthenticationError) assert response is None From a0bb771bf86ee53ecc3af4a8167cf2de2a160429 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Fri, 21 Aug 2026 08:10:19 -0700 Subject: [PATCH 14/14] Add utils as editable requirement --- .../tests/requirements.oldest.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt index fc4c7d834..fa900119b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/requirements.oldest.txt @@ -26,4 +26,6 @@ langchain-openai==0.2.0 langchain-aws==0.2.2 langchain-google-genai==2.0.0 langchain-anthropic==0.3.0 -boto3==1.37.0 \ No newline at end of file +boto3==1.37.0 + +-e ./util/opentelemetry-util-genai \ No newline at end of file