From b708def942ca3b37c0b4ed5cb2548c5f7c847a3b Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Sun, 16 Aug 2026 14:22:28 +0200 Subject: [PATCH] [opentelemetry-instrumentation-genai-smolagents] Add invoke_agent instrumentation --- .../.changelog/403.added | 1 + .../README.rst | 23 +- .../genai/smolagents/__init__.py | 42 +- .../genai/smolagents/_messages.py | 78 +- .../instrumentation/genai/smolagents/patch.py | 308 ++++++- .../tests/cassettes/agent_with_image.yaml | 76 ++ .../tests/conformance/agent.py | 71 ++ .../tests/conftest.py | 54 +- .../tests/fixtures/img.png | Bin 0 -> 3594 bytes .../tests/requirements.latest.txt | 5 +- .../tests/requirements.oldest.txt | 6 +- .../tests/test_agents.py | 849 ++++++++++++++++++ .../tests/test_conformance.py | 2 + .../tests/test_instrumentor.py | 207 ++++- .../tests/test_models.py | 33 - .../tests/test_utils.py | 135 ++- 16 files changed, 1725 insertions(+), 165 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/403.added create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/fixtures/img.png create mode 100644 instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/403.added b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/403.added new file mode 100644 index 000000000..9f302ffd9 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/403.added @@ -0,0 +1 @@ +Add ``invoke_agent`` spans for streaming and non-streaming ``MultiStepAgent.run()`` calls. diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst index edf44ac65..6dbb86098 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst @@ -36,10 +36,6 @@ for the client library instead: - the instrumentation or built-in telemetry of the client library the model calls (``huggingface_hub``, ``litellm``) -Agent runs (``invoke_agent``) and tool calls (``execute_tool``) are not -instrumented yet. A model call made inside an agent run still gets a ``chat`` -span, but no agent span sits above it. - ``TransformersModel`` is the only instrumented class with a ``generate_stream``. A streamed call gets a ``chat`` span that stays open until the caller drains the deltas. This covers both ``stream_outputs=True`` on an agent and a direct @@ -50,6 +46,10 @@ call also records the Known gaps: +* A managed agent called from the generated code of a ``CodeAgent`` runs in a + worker thread of the local Python executor. That worker does not carry the + calling context, so the managed agent's ``invoke_agent`` span starts a trace + of its own instead of nesting under the manager's span. * A subclass that inherits ``generate`` or ``generate_stream`` from one of the three classes above is instrumented. A subclass that overrides one is not: the override shadows the patched method, so the call produces no ``chat`` span. @@ -57,6 +57,7 @@ Known gaps: ``gen_ai.response.model``, no ``gen_ai.response.finish_reasons`` and no ``server.address``. A runtime in this process returns the generated text and the token counts, nothing more. It also listens on no socket. +* Tool calls (``execute_tool``) are not instrumented yet. Installation ------------ @@ -73,21 +74,13 @@ Usage from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) - from smolagents import TransformersModel + from smolagents import CodeAgent, TransformersModel SmolagentsInstrumentor().instrument() model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") - model.generate( - [ - { - "role": "user", - "content": [ - {"type": "text", "text": "How many seconds are in a week?"} - ], - } - ] - ) + agent = CodeAgent(tools=[], model=model) + agent.run("How many seconds are in a week?") Configuration ------------- diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py index fadfe4948..b88c6d998 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py @@ -11,8 +11,8 @@ ``MLXModel``) are recorded as ``chat`` spans. The API-backed model classes are not instrumented here: each one calls a client library that carries its own instrumentation, and emitting a span at this layer as well would duplicate the -span and count the token-usage and duration metrics twice. Agent runs and tool -calls are not instrumented yet. +span and count the token-usage and duration metrics twice. Tool calls are not +instrumented yet. Usage ----- @@ -22,21 +22,13 @@ from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, ) - from smolagents import TransformersModel + from smolagents import CodeAgent, TransformersModel SmolagentsInstrumentor().instrument() model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct") - model.generate( - [ - { - "role": "user", - "content": [ - {"type": "text", "text": "How many seconds are in a week?"} - ], - } - ] - ) + agent = CodeAgent(tools=[], model=model) + agent.run("How many seconds are in a week?") Configuration ------------- @@ -60,6 +52,7 @@ from collections.abc import Collection from typing import Any +import smolagents from smolagents import models from wrapt import wrap_function_wrapper @@ -69,11 +62,14 @@ from opentelemetry.util.genai.handler import TelemetryHandler from .package import _instruments -from .patch import model_generate, model_generate_stream +from .patch import ( + agent_run, + model_generate, + model_generate_stream, +) __all__ = ["SmolagentsInstrumentor"] - # The model classes that run inference in the current process. They call no # client library, so this instrumentation is the only place their model calls # can be observed. @@ -120,6 +116,7 @@ class SmolagentsInstrumentor(BaseInstrumentor): # only ``_instrument`` / ``_uninstrument`` rebind. _wrapped_generate_classes: list[type] = [] _wrapped_generate_stream_classes: list[type] = [] + _wrapped_agent_run = False def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -144,6 +141,7 @@ def _instrument(self, **kwargs: Any) -> None: self._wrapped_generate_classes = [] self._wrapped_generate_stream_classes = [] + self._wrapped_agent_run = False try: for model_cls in _model_classes_defining("generate"): wrap_function_wrapper( @@ -160,6 +158,16 @@ def _instrument(self, **kwargs: Any) -> None: model_generate_stream(handler), ) self._wrapped_generate_stream_classes.append(model_cls) + + # TODO: emit a span per agent step once the semantic conventions + # define an operation for one iteration of a reason-and-act loop. + # https://github.com/open-telemetry/semantic-conventions-genai/issues/81 + wrap_function_wrapper( + "smolagents", + "MultiStepAgent.run", + agent_run(handler), + ) + self._wrapped_agent_run = True except BaseException: # BaseInstrumentor.instrument() doesn't mark the instrumentor as # instrumented when _instrument raises, so uninstrument() would @@ -176,3 +184,7 @@ def _uninstrument(self, **kwargs: Any) -> None: for model_cls in self._wrapped_generate_stream_classes: unwrap(model_cls, "generate_stream") self._wrapped_generate_stream_classes = [] + + if self._wrapped_agent_run: + unwrap(smolagents.MultiStepAgent, "run") + self._wrapped_agent_run = False diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py index 129617c01..19859467f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py @@ -1,11 +1,12 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -"""Convert smolagents message and tool shapes into util-genai GenAI types. +"""Convert smolagents message, tool, and agent-run values to GenAI types. smolagents passes ``generate(messages=...)`` a list of ``ChatMessage`` objects -or plain dicts, and returns a ``ChatMessage``. This module maps those, and the -``tools_to_call_from`` tool objects, onto the types in +or plain dicts, and returns a ``ChatMessage``. ``MultiStepAgent.run`` takes a +task string and optional images, and returns a final answer. This module maps +those values, and the ``tools_to_call_from`` tool objects, onto the types in ``opentelemetry.util.genai.types``. util-genai then serializes them into ``gen_ai.input.messages``, ``gen_ai.output.messages``, and ``gen_ai.tool.definitions``. @@ -16,9 +17,11 @@ import base64 import binascii import logging +from collections.abc import Sequence from enum import Enum -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias, cast +from PIL.Image import Image from smolagents.models import get_tool_json_schema from smolagents.utils import encode_image_base64 @@ -34,7 +37,7 @@ ) if TYPE_CHECKING: - from PIL.Image import Image + from smolagents.agents import MultiStepAgent from smolagents.models import ChatMessage, MessageRole from smolagents.tools import Tool @@ -46,6 +49,9 @@ # ``image_url``, and a PIL image under ``image``. _ContentElement: TypeAlias = dict[str, Any] +# smolagents passes managed agents in ``tools_to_call_from`` alongside tools. +_ModelCallable: TypeAlias = "Tool | MultiStepAgent" + _DEFAULT_IMAGE_MIME_TYPE = "image/png" _DATA_URL_PREFIX = "data:" @@ -202,18 +208,21 @@ def to_output_message(output_message: ChatMessage) -> OutputMessage: return OutputMessage(role=role, parts=parts, finish_reason="") -def _tool_parameters(tool: Tool) -> dict[str, Any] | None: +def _tool_parameters(tool: _ModelCallable) -> dict[str, Any] | None: """Return the JSON Schema ``parameters`` object for a smolagents tool. A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it in an object schema, derives ``required`` from ``nullable``, and rewrites its non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the schema the provider receives. + + ``get_tool_json_schema`` also accepts managed agents despite its ``Tool`` + annotation. """ try: - schema = get_tool_json_schema(tool) + schema = get_tool_json_schema(cast("Tool", tool)) parameters = schema["function"]["parameters"] - except Exception: # pylint: disable=broad-except + except BaseException: # pylint: disable=broad-except _logger.debug( "Failed to build a JSON Schema for tool %s", tool.name, @@ -224,21 +233,52 @@ def _tool_parameters(tool: Tool) -> dict[str, Any] | None: def to_tool_definitions( - tools: list[Tool] | None, + tools: Sequence[_ModelCallable] | None, ) -> list[ToolDefinition] | None: """Map smolagents tool objects to function tool definitions. - ``Tool.validate_arguments`` runs on every instantiation and requires a - non-empty ``name`` and a ``description``, so both are read directly. + ``Tool.validate_arguments`` requires every tool to have a non-empty + ``name`` and ``description``. A managed agent must also have both. An + invalid entry without a name is skipped instead of recorded under an empty + name. """ if not tools: return None - definitions: list[ToolDefinition] = [ - FunctionToolDefinition( - name=tool.name, - description=tool.description, - parameters=_tool_parameters(tool), + definitions: list[ToolDefinition] = [] + for tool in tools: + name = tool.name + if not name: + continue + definitions.append( + FunctionToolDefinition( + name=name, + description=tool.description, + parameters=_tool_parameters(tool), + ) ) - for tool in tools - ] - return definitions + return definitions or None + + +def final_answer_parts(output: object) -> list[MessagePart]: + """Convert image and string answers without file-writing ``__str__`` calls.""" + if isinstance(output, Image): + if blob := _image_blob(output): + return [blob] + return [] + text = str.__str__(output) if isinstance(output, str) else str(output) + return [Text(content=text)] + + +def task_to_input_messages( + task: str | None, images: list[Image | str] | None +) -> list[InputMessage]: + parts: list[MessagePart] = [] + if task: + parts.append(Text(content=task)) + if isinstance(images, list): + for image in images: + if blob := _image_blob(image): + parts.append(blob) + if not parts: + return [] + return [InputMessage(role="user", parts=parts)] diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py index ff14ff493..ab06de3d1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/patch.py @@ -4,35 +4,46 @@ """wrapt wrapper factories for smolagents instrumentation. Each factory takes the shared :class:`TelemetryHandler` and returns a wrapper -suitable for :func:`wrapt.wrap_function_wrapper`, applied to the in-process -model classes only (see ``_IN_PROCESS_MODEL_CLASSES``): +suitable for :func:`wrapt.wrap_function_wrapper`: -- :func:`model_generate` wraps ``generate`` -> ``chat`` span. +- :func:`model_generate` wraps ``generate`` -> ``chat`` span, applied to the + in-process model classes only (see ``_IN_PROCESS_MODEL_CLASSES``). - :func:`model_generate_stream` wraps ``generate_stream`` -> ``chat`` span, held open until the stream is drained. +- :func:`agent_run` wraps ``MultiStepAgent.run`` -> ``invoke_agent`` span. -Original library exceptions are always re-raised unmodified; telemetry is -finalized via ``invocation.stop()`` / ``invocation.fail(exc)``. +Original library exceptions are re-raised unmodified. Telemetry is finalized +through ``invocation.stop()`` or ``invocation.fail(exc)``. """ from __future__ import annotations import logging -from collections.abc import Callable, Generator, Mapping -from inspect import signature -from typing import TYPE_CHECKING, Any, TypeAlias - +from collections.abc import Callable, Generator, Iterator, Mapping +from contextlib import ExitStack, contextmanager +from inspect import Signature, signature +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar + +from smolagents import AgentMaxStepsError +from smolagents.agents import RunResult +from smolagents.memory import ActionStep, FinalAnswerStep from smolagents.models import REMOVE_PARAMETER from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, ) from opentelemetry.util.genai.handler import TelemetryHandler -from opentelemetry.util.genai.invocation import InferenceInvocation +from opentelemetry.util.genai.invocation import ( + AgentInvocation, + GenAIInvocation, + InferenceInvocation, +) from opentelemetry.util.genai.stream import SyncStreamWrapper from opentelemetry.util.genai.types import OutputMessage, Text from ._messages import ( + final_answer_parts, + task_to_input_messages, to_input_messages, to_output_message, to_tool_definitions, @@ -40,6 +51,8 @@ from .provider import resolve_provider if TYPE_CHECKING: + from smolagents.agents import MultiStepAgent + from smolagents.memory import PlanningStep from smolagents.models import ( ChatMessage, ChatMessageStreamDelta, @@ -48,12 +61,42 @@ _logger = logging.getLogger(__name__) -# ``Model`` is quoted because a type alias is evaluated at runtime, unlike an -# annotation. +_InstanceT = TypeVar("_InstanceT") + _Wrapper: TypeAlias = Callable[ - [Callable[..., Any], "Model", tuple[Any, ...], dict[str, Any]], Any + [Callable[..., Any], _InstanceT, tuple[Any, ...], dict[str, Any]], Any ] +# Quote the alias because ``PlanningStep`` and ``ChatMessageStreamDelta`` are +# imported only for type checking. +_RunStreamChunk: TypeAlias = ( + "ActionStep | PlanningStep | FinalAnswerStep | ChatMessageStreamDelta" +) + + +@contextmanager +def _recording(what: str) -> Iterator[None]: + """Suppress extraction errors so telemetry cannot break the call.""" + try: + yield + except Exception: # pylint: disable=broad-except + _logger.debug("Failed to record %s", what, exc_info=True) + + +def _finish( + invocation: GenAIInvocation, error: BaseException | None = None +) -> None: + """End the invocation. An interrupt ends the span without an error.""" + if isinstance(error, Exception): + invocation.fail(error) + else: + invocation.stop() + + +# Keyed on the underlying function: wrapt hands the wrapper a freshly bound +# method per call, so keying on that would retain every model and agent. +_signatures: dict[object, Signature] = {} + def _bind_arguments( wrapped: Callable[..., Any], @@ -63,12 +106,21 @@ def _bind_arguments( """Bind call args to the wrapped callable's signature, applying defaults. smolagents passes the interesting arguments positionally - (``model.generate(input_messages)``), so binding is what makes them - readable by name. On a binding failure the keyword arguments are returned - on their own, without the positional ones and without the defaults. + (``model.generate(input_messages)``, ``agent.run(task)``), so binding is + what makes them readable by name. On a binding failure the keyword + arguments are returned on their own, without the positional ones and + without the defaults. """ try: - bound = signature(wrapped).bind(*args, **kwargs) + function: object | None = getattr(wrapped, "__func__", None) + call_signature = ( + _signatures.get(function) if function is not None else None + ) + if call_signature is None: + call_signature = signature(wrapped) + if function is not None: + _signatures[function] = call_signature + bound = call_signature.bind(*args, **kwargs) bound.apply_defaults() return dict(bound.arguments) except (TypeError, ValueError): @@ -219,14 +271,7 @@ def _record_request( args: tuple[Any, ...], kwargs: dict[str, Any], ) -> None: - """Record the request on the invocation. Extraction errors are dropped. - - The messages, tools and keyword arguments come from the caller, so the - conversion can get a shape it does not handle. The span is already - started and the model has not been called yet, so an error raised here - would both break the call and leave the span unfinished. - """ - try: + with _recording("the chat request"): bound = _bind_arguments(wrapped, args, kwargs) _apply_request_parameters(invocation, instance, bound) invocation.tool_definitions = to_tool_definitions( @@ -236,8 +281,6 @@ def _record_request( invocation.input_messages = to_input_messages( bound.get("messages") ) - except Exception: # pylint: disable=broad-except - _logger.debug("Failed to record the request", exc_info=True) def _record_response( @@ -245,17 +288,10 @@ def _record_response( invocation: InferenceInvocation, output_message: ChatMessage, ) -> None: - """Record the response on the invocation. Extraction errors are dropped. - - The model call has already succeeded at this point, so an error raised - here would turn a completed call into a failed one. - """ - try: + with _recording("the chat response"): _apply_token_usage(invocation, output_message) if handler.should_capture_content(): invocation.output_messages = [to_output_message(output_message)] - except Exception: # pylint: disable=broad-except - _logger.debug("Failed to record the response", exc_info=True) def _start_inference( @@ -280,7 +316,7 @@ def _start_inference( return invocation -def model_generate(handler: TelemetryHandler) -> _Wrapper: +def model_generate(handler: TelemetryHandler) -> _Wrapper[Model]: """Wrap a defining ``Model.generate`` to emit a ``chat`` span. An in-process runtime returns the generated text, the token counts it made @@ -361,10 +397,7 @@ def _finalize(self, error: BaseException | None = None) -> None: output = self._output_message() if output is not None: invocation.output_messages = [output] - if error is not None: - invocation.fail(error) - else: - invocation.stop() + _finish(invocation, error) def _on_stream_end(self) -> None: self._finalize() @@ -374,7 +407,7 @@ def _on_stream_error(self, error: BaseException) -> None: self._finalize(error) -def model_generate_stream(handler: TelemetryHandler) -> _Wrapper: +def model_generate_stream(handler: TelemetryHandler) -> _Wrapper[Model]: """Wrap a defining ``Model.generate_stream`` to emit a ``chat`` span. ``stream_outputs=True`` routes an agent's model calls here. The span stays @@ -388,11 +421,196 @@ def wrapper( kwargs: dict[str, Any], ) -> _ModelStreamWrapper: invocation = _start_inference(handler, wrapped, instance, args, kwargs) + with ExitStack() as finishing: + finishing.enter_context(invocation) + stream = _ModelStreamWrapper( + wrapped(*args, **kwargs), invocation, handler + ) + # Transfer invocation finalization to the stream. + finishing.pop_all() + return stream + + return wrapper + + +def _record_run_answer( + invocation: AgentInvocation, + agent: MultiStepAgent, + output: object, + *, + capture_content: bool, +) -> None: + # Reaching ``max_steps`` returns normally with ``AgentMaxStepsError`` on the + # final ``ActionStep``. + steps = agent.memory.steps + last_step = steps[-1] if steps else None + finish_reason = ( + "length" + if isinstance(last_step, ActionStep) + and isinstance(last_step.error, AgentMaxStepsError) + else "stop" + ) + invocation.finish_reasons = [finish_reason] + if capture_content: + invocation.output_messages = [ + OutputMessage( + role="assistant", + parts=final_answer_parts(output), + finish_reason=finish_reason, + ) + ] + + +class _AgentRunStreamWrapper(SyncStreamWrapper[_RunStreamChunk]): + def __init__( + self, + stream: Generator[_RunStreamChunk, None, None], + invocation: AgentInvocation, + agent: MultiStepAgent, + *, + capture_content: bool, + ) -> None: + # Model-stream metrics do not apply to agent steps, so do not pass the + # invocation to the base class. + super().__init__(stream) + self._self_agent_invocation = invocation + self._self_agent = agent + self._self_capture_content = capture_content + self._self_final_output: object = None + self._self_saw_final = False + self._self_finished = False + + def _finish_once(self, error: BaseException | None = None) -> None: + if self._self_finished: + return + self._self_finished = True + _finish(self._self_agent_invocation, error) + + # The base wrapper finalizes on StopIteration and on an Exception, but not + # on an interrupt, which would leave the span open. + def __next__(self) -> _RunStreamChunk: + try: + return super().__next__() + except BaseException as error: + self._finish_once(error) + raise + + def close(self) -> None: try: - stream = wrapped(*args, **kwargs) - return _ModelStreamWrapper(stream, invocation, handler) - except Exception as error: - invocation.fail(error) + super().close() + except BaseException as error: + self._finish_once(error) raise + def _process_chunk(self, chunk: _RunStreamChunk) -> None: + if isinstance(chunk, FinalAnswerStep): + self._self_final_output = chunk.output + self._self_saw_final = True + + def _on_stream_end(self) -> None: + # Recording errors must not replace the stream's StopIteration. + with _recording("the agent run"): + if self._self_saw_final: + _record_run_answer( + self._self_agent_invocation, + self._self_agent, + self._self_final_output, + capture_content=self._self_capture_content, + ) + self._finish_once() + + def _on_stream_error(self, error: BaseException) -> None: + self._finish_once(error) + + +def _record_agent( + invocation: AgentInvocation, + agent: MultiStepAgent, + bound: dict[str, Any], + *, + capture_content: bool, +) -> None: + with _recording("the agent"): + invocation.agent_description = agent.description + # Managed agents are exposed to the model as tools. + invocation.tool_definitions = to_tool_definitions( + [*agent.tools.values(), *agent.managed_agents.values()] + ) + if capture_content: + invocation.input_messages = task_to_input_messages( + bound.get("task"), bound.get("images") + ) + + +def _record_agent_run( + invocation: AgentInvocation, + agent: MultiStepAgent, + bound: dict[str, Any], + result: object, + *, + capture_content: bool, +) -> _AgentRunStreamWrapper | None: + """Record a finished run or wrap a streamed run until it is drained.""" + if capture_content and bound.get("additional_args"): + with _recording("the agent task"): + # ``run()`` appends ``additional_args`` to ``agent.task``. + invocation.input_messages = task_to_input_messages( + agent.task or bound.get("task"), bound.get("images") + ) + + if bound.get("stream") and isinstance(result, Generator): + return _AgentRunStreamWrapper( + result, + invocation, + agent, + capture_content=capture_content, + ) + + with _recording("the agent run"): + # ``return_full_result`` can be set on the agent, so ``run(task)`` can + # return ``RunResult``. + output = result.output if isinstance(result, RunResult) else result + _record_run_answer( + invocation, agent, output, capture_content=capture_content + ) + return None + + +def agent_run(handler: TelemetryHandler) -> _Wrapper[MultiStepAgent]: + def wrapper( + wrapped: Callable[..., Any], + instance: MultiStepAgent, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + agent = instance + bound = _bind_arguments(wrapped, args, kwargs) + capture_content = handler.should_capture_content() + # Only managed agents require names. Use the class name for unnamed agents. + # Models may omit ``model_id``. + invocation = handler.invoke_local_agent( + agent_name=agent.name or type(agent).__name__, + # ``gen_ai.request.model`` applies because an agent uses one model. + request_model=getattr(agent.model, "model_id", None), + ) + + with ExitStack() as finishing: + finishing.enter_context(invocation) + _record_agent( + invocation, agent, bound, capture_content=capture_content + ) + result = wrapped(*args, **kwargs) + stream = _record_agent_run( + invocation, + agent, + bound, + result, + capture_content=capture_content, + ) + if stream is None: + return result + # Transfer invocation finalization to the stream. + finishing.pop_all() + return stream + return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml new file mode 100644 index 000000000..d417ed7c3 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/cassettes/agent_with_image.yaml @@ -0,0 +1,76 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: '{"messages":[{"role":"system","content":[{"type":"text","text":"You are + an expert assistant who can solve any task using tool calls. You will be given + a task to solve as best you can.\nTo do so, you have been given access to some + tools.\n\nThe tool call you write is an action: after the tool is executed, + you will get the result of the tool call as an \"observation\".\nThis Action/Observation + can repeat N times, you should take several steps when needed.\n\nYou can use + the result of the previous action as input for the next action.\nThe observation + will always be a string: it can represent a file, like \"image_1.jpg\".\nThen + you can use it as input for the next action. You can do it for instance as follows:\n\nObservation: + \"image_1.jpg\"\n\nAction:\n{\n \"name\": \"image_transformer\",\n \"arguments\": + {\"image\": \"image_1.jpg\"}\n}\n\nTo provide the final answer to the task, + use an action blob with \"name\": \"final_answer\" tool. It is the only way + to complete the task, else you will be stuck on a loop. So your final output + should look like this:\nAction:\n{\n \"name\": \"final_answer\",\n \"arguments\": + {\"answer\": \"insert your final answer here\"}\n}\n\n\nHere are a few examples + using notional tools:\n---\nTask: \"Generate an image of the oldest person in + this document.\"\n\nAction:\n{\n \"name\": \"document_qa\",\n \"arguments\": + {\"document\": \"document.pdf\", \"question\": \"Who is the oldest person mentioned?\"}\n}\nObservation: + \"The oldest person in the document is John Doe, a 55 year old lumberjack living + in Newfoundland.\"\n\nAction:\n{\n \"name\": \"image_generator\",\n \"arguments\": + {\"prompt\": \"A portrait of John Doe, a 55-year-old man living in Canada.\"}\n}\nObservation: + \"image.png\"\n\nAction:\n{\n \"name\": \"final_answer\",\n \"arguments\": + \"image.png\"\n}\n\n---\nTask: \"What is the result of the following operation: + 5 + 3 + 1294.678?\"\n\nAction:\n{\n \"name\": \"python_interpreter\",\n \"arguments\": + {\"code\": \"5 + 3 + 1294.678\"}\n}\nObservation: 1302.678\n\nAction:\n{\n \"name\": + \"final_answer\",\n \"arguments\": \"1302.678\"\n}\n\n---\nTask: \"Which city + has the highest population , Guangzhou or Shanghai?\"\n\nAction:\n{\n \"name\": + \"web_search\",\n \"arguments\": \"Population Guangzhou\"\n}\nObservation: + [''Guangzhou has a population of 15 million inhabitants as of 2021.'']\n\n\nAction:\n{\n \"name\": + \"web_search\",\n \"arguments\": \"Population Shanghai\"\n}\nObservation: + ''26 million (2019)''\n\nAction:\n{\n \"name\": \"final_answer\",\n \"arguments\": + \"Shanghai\"\n}\n\nAbove example were using notional tools that might not exist + for you. You only have access to these tools:\n- final_answer: Provides a final + answer to the given problem.\n Takes inputs: {''answer'': {''type'': ''any'', + ''description'': ''The final answer to the problem''}}\n Returns an output + of type: any\n\nHere are the rules you should always follow to solve your task:\n1. + ALWAYS provide a tool call, else you will fail.\n2. Always use the right arguments + for the tools. Never use variable names as the action arguments, use the value + instead.\n3. Call a tool only when needed: do not call the search agent if you + do not need information, try to solve the task yourself. If no tool call is + needed, use final_answer tool to return your answer.\n4. Never re-do a tool + call that you previously did with the exact same parameters.\n\nNow Begin!"}]},{"role":"user","content":[{"type":"text","text":"New + task:\nDescribe what you see in this image briefly."},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAYAAADDPmHLAAAQfElEQVR4nO2deXRUVZ7Hv7/3KpXKDoEEApFAcGMJ0LKL7AKiIoOSqI1IQMXT2tN9TvfMdKNIpZpuenB6PGdae47SLMHdFIILuIyySNhEgkAQcAGMQshCCmP2qnr3N+fFIBAIZKl6775Qn3+gUsu9Vff77rv3d38LoZ3hdDqV/O/QUxVKH1JwPZh7AOjOQHcAHQCOAxALUMx5b/MBXAmmKhAqAZQAfIqJisH0DQS+oTDxdd9kHHO5XALtCILFufthZ7LqpzEADwNhGBiDAIoISmNcL459rGAvGDsJvMWd7SqChbGcAKb+69/Do8rPTIAi7iDQRAA3mtylIwT+iKG8iyrxidvt8sJCWEIA45xOW2KBMlWwuJ+I7vhpCpeSHwn8rhDKK6d7iY+2uFx+SI7UArgnc/F1KmmPMuMBAF1gLUoAXq2w7YU3Vj91FJIiowBo5pysSUT4LYDbACiwNgLEHxLwTM4q18eQDJkEQBmZWdMZvBCgwWiPMPYw8V/XZLvW1T+SACkEMHNO1mQiLAX0FfzVAOcBygJ3tvOjq1oA9z6U1V9o/DeApuCqhD/UCI+tXeU6dlUJID39mQiKKv8DgxYAsOOqhmsIeJqrsMSMLaThAsiYkzWGCasB9DS6bck5AMGPuF907W6XAtD38p0LsIiAJwBSjWrXYgiAn62M6fSH95/9TV27EUD6bGcPKPQKCLcY0V47YC80dab7paeOB7uhoO+xZ2Zm3QcVB0KD3yJugur/LGOua4qVZwDKyHQ6GeQMYhvtHWbw0/1S8ESwTiGDIoD0dKedI2kFUb0JN0TbeRdVsfe63b+rgewCuGve0phwUZPTYMYNETi22n087ZVXXD9KK4D75jm7aYI2XD0WPaPhPJudbnttWdZp6QTQMPi5AFID9ZkhLgHxYWhhk9wvLjwJWQRw//yszn4vPgHQF/LhDbMp38XGRp9O6BxXFxMdJRwOGzkc4WpMTFS9PaKiokqrra3Tamv9XFFZpZSeLg8v/7Eqwe/XrpHUUnlMs/HYtctdJ0wXwKxZzlhvGDZJdIJXHdch+uAvBvSuGTG0X3KXLh1TVEWxteaDNCH8Rac8BTv3HDqx/8DXUeXl1brAIyEHB+w+Ht3WNUGbBDBtflakw8sfADQaJmO3hx2eNGHw6Ynjbhpqs6mOYLTh82k1H2/Oy9u4JS/B6/XfALNhfNwxPOn2Zcse9RkuAN379lABvQNAd9EyDbvd9uXsX06pHtg/9RdGtvv5/qOfv/z6h9E+n3YdTIQIL+asyspsrX9BqwWQPjfLBcYimAbXjBqZtmvmjLGjWzvFtxVNE7431mzaseuzw8MBBGXWaQ4MXrwm27XIMAHMfNA1lRReb5a7Fil0+rH500/ccO01Umw3Dx/57ovnV7yTxMzxZvWBQRlrsp3uoAvgvkxnTw2UB8CUL6sqyqkF/z7Ll5jQQQ/4kIaikjMFf/2vlyOYkWhOD/gMNAxyv+T6riXvatEVnJnpdGhMusrMUTpz5W8eu6dMtsHX6ZrYMeW3j99zhkBVMAXqqJ+4pqfnqEETQDXwNAhDYBLT7hi1r1fPrv0hKak9u90wdcqwfaZ1gHALRx1eGBQB3JvpHMGgx2ESMTGReyZNGCy9P8GUW4eOio2J+Nys9ol5Yfo85y0BFcD8+S+ECWCZiT76PH/unVGwAESEeQ/eYV5fCTYIytZD6Jrz8mYNqMdX9G8ApcEkYmMi81J6dOkDi5DaK+l6M2cBAL2jKs/8LiACSH/IeS2xeAomMmH8YEP84wLJ2NGDAn523+Jbge6K1+YZQKN/BC3cunl4Rw3vOxAWY9TNA9LAMDM4NJJULGmTABp80ibDRBwO+1fh4fZoWIxIhz3G7ggzNSiUQb/U3fBbKwBi5isqKNh0TexYBouS2CkuYI4brYQY+O/LGfyaFEDGHNfd9d6pJpOQGG/ZlCyJCfHm5wcgDEmf47q9xQJg8B8hARGOcCkCWFtDZKQkfaemF/GXFEB6pmuSmRa/83GE2yybHyA8wi6HAEDD0+dmTbjUM038uNysPaQR1Nb6LHsLqKvxSpEDoAE94caVBXDvnMW9zV75n4/fb/5ttLUIoUEaGHc2jO3lBcCK9qt2kJYlxMUoGvkfueiPjSN4GxIyhWiHEGh24+PiCwSQWFA/9VstG1eI5tONog/f2qQAGDSrBR8WwoII5vsvKYCfjg95mim9CmEYBEzXg3cvEkBM5ZlxjRIoh2ifdKBojLlIACz4TtO6FMJYuD7dbqM1AMmz9w8RXBisJ9k+J4C7H/hLEoDrg9xuCGmg/g1j/pMAFJtvrNldCmEopNj8t5y7BRBGGtt+CNNhHvazAIhZltDuC1BV5bupU4abXRCi1dw2ecQNqqp8DwkhYj2eEUqDaVCKGLtLuIJ74mKjusKidIiL6vJw5h0/yJIZ/AKY9GhqUkTUl6kASedzHx3t2NfnxhQZhdki+vXpmRYV4TgA2SBEp892XqOo8Jsa394UA/qllqOdkJaWqs8C0kE2pY8CpmshIV26dpIxN0+rSOoaL+V3EYzrFSaWMqtXuM26rmCNsdvtUn4XhThZYaJ6g0CIqw9mJCvEsOwqO0Rb4e761GRSRosQEtBBkbgIY4igQ3GKRIkPQxgOhwRwlaNvTzhUv+eqhVQFIImiF85R47VuRFBj/F6/fGcBP6HoM4CUAjh1ymOpMuyXo7C4TNIMJ6zpi0ApY68OfnHUtKybgSb/4NEOkJNKXQABLUESKKpr6gYcOvKtfKdoLeSLw9/mV1bVDpRZAB5IyrKVGzqV/1hdAovyww8VJcuzN3SEvMgtACFEd+efV/qLS34ogMXIP3R8f9aS1X5NE8mQFirRj4NLITFCcLf1H+wIegXNQCEEa09mLf/8nyvXD9T7DrkpVBj4FpLz1ZffJ8Ai7Nv/zb6KyhpDi1e0gZOKApJeADV13j4VVTVmZ9xqFh9s/KwWFoGYTlhiBtANFltz9x+C5NTWeauKik5bxo9RAEcUTRGHYQFytx2QeTVdz/r3d+bJ6GDbFDZVHFbWrnIdB1h6B8zq2rq0wlNlpmbevNLib/uOg71gGbj89ZWuQn0byADthwVwr9vS5kKJwWLHzi/2aELohSatAdFe/Z8GZ0U2M7V5szl67OSQiqpq6VLHMrN4671tspp7Lwkz7z4vNEzRa/5aAIrKefMT6czDG7fk7fLW+cwvJNkyzgnAJ1gXgKxHlhewP//okIqKGmnMw36/8G54/9MUWAsOs9m3/yyAdS9l6T+oJXYDYI5Z+eJ70vT1FfdHuzQhusNa7H9t+ZPF+n9+Dlgg4P9gEY4eL7y5oKD4S7P7cbKw7Hhe3lf1UbZWgujcWJ+LWBH1dYCtQthzL6zT/EK0umhyIBZ+zz7/ZqUexASLwcCHFwmgpJfIlflksDF1Xl/fHPemHWa1/1rOxm3V1XWmFdJqPVxa2oO3XiSALS6Xnwh6PWDLsOuzQ8NOnjx9zOh28w8e37dr9+GbYUEYeEsf67OPGwUt8kuwFBTxzHNuoU/HRrVYU1X7w/LVG7rX1+ezIATlggLTFwigTw9sAljKlCZN4fP5r9XYOMfWM+VVHma2zPH0BTBOoKrPpiYF4HK5BDFeNrxjIYxBwUq3O+OCi+WiuHW/LewFWV3FQ7QJoSm2lY3/eJEA1q5YWADQ221rK4R0EL/909heyCUzVxDjfwzpVAjDIKHq9QPRLAHkrM7aCtSfD4RoH+zOWb2o3vbfmMvkrlEWB7FDIQyEwK6mnmtSAO5s50cAX1I1kiFUIsOSMDGxJLUAmwljW062672mnr7sD8dQF0ByFKJTRGRYiHtURLhlfP50mJQnLvf8ZQWwJntRLgEXWI5kIzo28pSR7cXFRScQ6AysAPE6fQwv95IrTp0K2X6v+2RCUm4de5OhJ4JERKm9uuVDergG/vqxQ5sE8Pqqhd+DeSkkRLWpx0ePGmB4jeOM9PEpYDnD6s/CUJa6X3rqiiF1zVs8VeM/wTgIiSBQ1e9/naGpqhJmdNtJiR1Txo4ZZNpRdDM4Eg3RrIu2WQJwu11eoSgPy2IiVhTl5K8emf5tcnJn0/Ic3zN99Jh+fVO3QDpYE6RkZme7mhWi1qItTcYc5xImMm9nwPCnpHTZ+dhDd6VFRDmkcMPenXdkzxvuzfE+v1+KnMsMXrom2/XH5r6+RWfaJT2xKKEAen0hw50hYqIj985/aFpMyjWJoyERwwbfOGToTTeI3O35O99ev72LqUJg7KFqLGrJW1ps1EifvbgXVP/nepZJGIDDYT8454HJ/n439pI+6FIIFpu37v/svQ92JhgvBC7XCDetXeVqkYdUq6xaM+c67ySuPzEMmgXOpioF/zJtdOHoUWkj9K0XLIQwXgjM4Jlrsl1rW/rGVv+wGZnOJxj0FwQYIniGD+mTnzFz/AibqlrO4/Z8dFe13O35nwb91sDsdK92/ak1b23LlUUZmVmvMnAfAgJXpfVP3fXA/VOGRoSHtasE1iKIMwITctasyrqvtZFdbXFs5IqY+MzoCo/uHzex9Z8Cf/I1idvmz72zT4e4qNZ/jsQoCikTxw0aPn7MQF0InwZOCJxLlXGZbQnra/O99a55S2PCRfVmgAa3YWUvZeEquW8NnB/mdYx99dUFbTqXCMjiasbsrESbgo0g9G9vK3s5bw38NYAx7mxXUVv7ELDVdfrcJQlg78d6xbemXqOqyrczpo0+ZcWVvTwzAn8JETbR/eLCk4FoO6CDUC8C1H3YUJXyXCNEZePHDPpi2u03jzTDdm8VhGBt89a9uzZ8sDvJf0khcL7NZp90NrI3EAT8Kpw1yxnrtdGbINyqP+7WrfO2Xz86o290lKPdJH8ONpoQ/nXv5O7I3Z4/kJkbDG6cG+Z1TG/rPb8xQZmG09OddiVa/eeUSUNTp04eVl+mPETLKSsrL3z6mdfP1NZ5D0WCH2zuAU9LCNp9mJmVIo9nIRjOYFoM2zns8/n/vOL5/83So7aC0UDQF2KnSkvHg5TX9GqwwW6rnVEGQQ8mJcY36dAZCAxZiZ8oK0tWGTkARhrRXjsgj2xqetcOHYKeJNuQqTm5U6cTZZ3ixwH1rmVSu1KZjAbC38o6xd9sxODrGL4XL/Z4BgjBKwAY7ssnNYwDIH4kqXPn+vRtRmGKMYaZbcUez+PM0E8TLeVnHwRqwVha1jl+SX8iwwtlmWqNKykpuU5T1CW6i53ZfTEBPcToLUVo/5GYmPgNTEKKH73I40ljwU/pJgRcBRBju2Ba0C0x3vQAXCkEcJai06cnNDiZjED7ZCdYPJmUkLAZkiCVAM5SWFo6WFGU+cyYDSAC1sYL4G0CL+vaubN+WCYVUgrgLMXFxV2EGpYJ8GMAesBaFAG8WiN6Tt8GQ1KkFsBZmFktLisby1DuBniGfsYEOSkE+C0C1nbp1OkTIpLe5mEJATQ+YyjxeIYLfefAmAygn4lnDQLAIRDeV4B1ifHxnxKRpYpeW04AjfF4PHG1zCNJYCQIowDoyZujg9RcBQOfEmMHK9jpINoZHx8vfbmddi2AS1FaWtrNq6qpihC9wZRKxL0ZlKSH9zcYnvSFZex5QtGTPlc0hMFXAfQjQZxkpmMgPsZCORZG2tGEhIRCk79awPl/+oMNTPkSoNoAAAAASUVORK5CYII="}}]}],"model":"gpt-4o","stop":["Observation:","Calling + tools:"],"tool_choice":"required","tools":[{"type":"function","function":{"name":"final_answer","description":"Provides + a final answer to the given problem.","parameters":{"type":"object","properties":{"answer":{"type":"string","description":"The + final answer to the problem"}},"required":["answer"]}}}]}' + headers: {} + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DVJPdiwp16bKXEYMV5LTiliNkq9Rf\",\n \"object\": + \"chat.completion\",\n \"created\": 1776355161,\n \"model\": \"gpt-4o-2024-08-06\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_UemzWkeWxboyk7wCbBNRfUZp\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"final_answer\",\n + \ \"arguments\": \"{\\\"answer\\\":\\\"The image is a generic + user avatar icon, typically used as a placeholder for a profile picture. It + features a silhouette of a person against a circular background.\\\"}\"\n + \ }\n }\n ],\n \"refusal\": null,\n \"annotations\": + []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 1102,\n \"completion_tokens\": + 44,\n \"total_tokens\": 1146,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_07a5e8f420\"\n}\n" + headers: {} + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py new file mode 100644 index 000000000..2cffd623e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conformance/agent.py @@ -0,0 +1,71 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import pathlib +from typing import Any + +from smolagents import OpenAIModel, ToolCallingAgent + +from opentelemetry.instrumentation.genai.smolagents import ( + SmolagentsInstrumentor, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class AgentScenario(Scenario): + # The model call goes to the OpenAI SDK, whose instrumentation is not + # enabled here. Tool calls are not instrumented yet. + expected_spans = {"invoke_agent": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + from PIL import Image + + image_path = ( + pathlib.Path(__file__).parent.parent / "fixtures" / "img.png" + ) + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("agent_with_image.yaml"): + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + agent = ToolCallingAgent(tools=[], model=model, max_steps=3) + agent.run( + "Describe what you see in this image briefly.", + images=[Image.open(image_path)], + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + agent_names = { + attr["value"] + for entry in report["samples"] + if "span" in entry + for attr in entry["span"]["attributes"] + if attr["name"] == "gen_ai.agent.name" + } + assert "ToolCallingAgent" in agent_names, ( + f"expected the agent name on the invoke_agent span, saw {agent_names}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py index 01ae6a97d..12cff5b0f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/conftest.py @@ -11,10 +11,43 @@ SmolagentsInstrumentor, ) from opentelemetry.test_util_genai.instrumentor import instrument +from opentelemetry.test_util_genai.vcr import ( + scrub_response_headers_overwrite, +) + +from .test_utils import LifecycleRecorder -# No VCR plugin: the instrumented model classes run inference in this process, -# so there is no HTTP traffic to record. -pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] +pytest_plugins = [ + "opentelemetry.test_util_genai.fixtures", + "opentelemetry.test_util_genai.vcr", +] + + +@pytest.fixture +def lifecycle(tracer_provider) -> LifecycleRecorder: + recorder = LifecycleRecorder() + tracer_provider.add_span_processor(recorder) + return recorder + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers_overwrite( + { + "openai-organization": "test_openai_org_id", + "openai-project": "test_openai_project_id", + "Set-Cookie": "test_set_cookie", + } + ), + } @pytest.fixture @@ -52,3 +85,18 @@ def instrument_event_only(tracer_provider, logger_provider, meter_provider): emit_event=True, ) as instrumentor: yield instrumentor + + +@pytest.fixture +def instrument_span_and_event( + tracer_provider, logger_provider, meter_provider +): + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_AND_EVENT", + emit_event=True, + ) as instrumentor: + yield instrumentor diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/fixtures/img.png b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/fixtures/img.png new file mode 100644 index 0000000000000000000000000000000000000000..d412166079109b2796fd9ccd719263a974087cc3 GIT binary patch literal 3594 zcmV+l4)yVgP)8*ZnHXbN#mejsK~x`h%IdRDNml~HXImAJiph?I6bx^tcMn^`!YO{4vA$6!%wT$QO45bl-N2!QXdoqb}de?tuW|U zj9ufD)JGg5%i#w9RL2Qa_>>1jHUuc78Rx^+6!n;tc9&CDR&&Gl=op(iUjB*5h!fMk zkMm_~e9du)%CETr-{bV_7}jd^^RmhNPMJM`GiqzJ=J3+`ANW(_)-hsqASzY7>=Fxk z#hIVg8hPStGzN=WAjs7Ru^nce;*ymgaAWM(8fdn__CuQmwoBAY5Y}AJOUnv3!g;Mx zW*e+;$QHAu3h5M;U-FEf&>CXaQzy)4cVl}+ji6r43qUR&@t1TmZ!=6kv}i(O7q*|J zCCp>rfhJU)4BO~aEzC>19ox~88fh~>t&>TcabW?cto{bu(@L5%Qqa@MoXsQ|;GfME zmg|Lt5-qRWZ`R3>&Gh&W)SR%LN$Sz0l{fGN@6*YI&CDoCOk_7=dy`Up61zp1RV{i( z!`yRIm9JyFlhBCsTeXR{2`e&d6y~SjitTTzAS#8S;Z`~cHlOneDX!xS{<`MFe=mEz zIAh(m!gU*W7H!=0PU+U&g%tq&4gU|0!8vg5+?2Ey_^rgz4XTJ;_8OxmQJ7lq9+g>oDT3?cipj;%-44pr!2~-f@|yC&n)UpFB3eHs(ghz zzf>o2hLf1OwDGNN`%61|eCMzu!^J6&ukVeVePxNM=|}PXO+i!+rlxNA684KBg1RYB z#JieA84NgMe{t1e?5s%he!+haCp?kXjMHN@PFZ<7#*Fa=f~=@GF~-9BUf8g+a3-(# znbPU?UwHnd?ZrGj=d|fF4(MaS{)woD#>tyFT+5lH`im=TYp_GiqOiQ?h*MUBIGsk~ zkW>NtDI$uO*!TDiP0MV2lB>I}skI$D!E9<8TTb%Vu@|S;I9Lce-LSvHDoJA3!MHZ< z%?(0iXAikQZotN;LWR5Gs8dpVHR(1A=y4Ttuf~20n}#Pq=-k*KHsu~D!$zmZn_Ksl z<8&Jd{}eFnx3DNqVm}p3{O%_$%&5Y~rbxFxFqph>fHDSMzSTZpd1^RI-7Dofa&O(yco!5VW}1H3w@w-aU(f@vdeQ88<;w+ot`s8-(zil z3nv0o0M%MO1Y*?$Xd~FW&6KL{eIHLM6_V0u;wZ-oW zGGE(MgbiyIUfi&&1n0*bSS9xFxEfmSL~{{ycy;r;CD^c5;gwDAkQgg5gsj|${XC`w zLZf;!U)^1V4QmzXya5ylfW6qyx`HK51QVNlF>BZFD8h!d3TxNr(g^_W(@8B?GqNTR z{;_G;0T4c5mJ#;%D8UR~f!Kmfb^wHxu5r7Iof7+<&W}vR4uJ47?u1i*!WR4kv#|pp zyf`G~E$sIpLHD7cT|?J&>;MQmpXe?}L!sSY4?6(DkM5dqh-Xlgyx!}D+o=ZglcblCYq?$!Js=FMgq zn}Nde!(`_FO_%(#Y8XyGpp!py5tXwsFj5FMWzZZXfc(q~!>4>l3hAj8wK#tyfKkH~ z+y4Rn{Vc+Uv?|ZPoJ(V97-Fbjz$q$!#s?^nv%8=I8`7%eVDLQz0J)LCfcVt%I&4U*B0o*E zGe}}S2!KxQ0U8vPS6e;^M!JJFd7}Yqg8(3#z_=t==h3zfFE*-GXlgm>jZb!U;C#`6 z)k6Rf{XPv7m!x*pHnyI`#-+}IQgxL|obHaImhU&|vowqh(` zuSXaeK1Z;~?%uEs95y-?pzs%;M0E~;9_;iL6GLc@3c@UQZ6v#D=#IT5R6Bh1k(I(L z0)XbgP%W&txxFKRjc+#H{y;y0b%CeEs0E{_EDJjs(${2@TRL|04%3&0pE5iEVlM~4 zALl9G6evz#(nD6yG@s}s(>&bK<3G>8{bc)7#$aG>%F>%I2LMQ*yZN(*#VVb$z>3Tq zOO&4wHKUvlf@MQ=PNd3(%Ie0W*cnFSu5WJRY580yeWV$h5Q62LuilGKq|D=~OF!=L z`7dCn7zGsk#6K-3+mkvjdJ9L7tP*P4j-b;4_U*-vF$#WfE57;%VGs!T8vB1puoJcR z2b5ub?z&^G*g28rK(V{V?EQeI3ord|#D2dlSU(Q`fI10Jq#uXZF?Ld;RY;r|9}Q z_Vc*np8+~MUQK`<02A^N_V0wEv~}D>*a0v;uo4{h@1!IUT|pGl-|Y`v!j6g*?uKK~ z{*Ti@7f~t0ex0-&lIs_m3lbwvRy(niA_>g)!D*lj_z?SbN^(j{F1{!wk{7M2!cK}L z{B1Mfl++MT2T7(<@*hj_a{9OV#Xi$qoCsU<3Jzf>MViIxs}|@8VtvBAwA-;irzJsT zOSCsIpZJutwob2c1;8&j+}LuGdS{SBQW^H+v<16N@eDWMo0tiI4fj^n9y^YmV-#7h z@1gt_Xp$s$9rojl^qB1W36&;dm^NNiQEU2qK_z2T5l)-=uuI@8?7y(c^o}C_+Wp&- z^`+1yi=AN>dq1eCN=Qj3(>5^S{=hTQ$fzMxhCnAhlTlq+)6|R|Kmiz!uzgR_A=3jv zRH}40g$$>lWNGA7vUv-ZHRPAOEwOV*moLzty|bXwoIqfwVV)=US48BH+3)#H`?ZM( zA9IskC*i4rdN(#Sbxa6^fl(7ZjF3}e&*>y&#yDjq2!lO4yuQz|VXcN?p9Wq{$Nn)w zbDWa0Rg;)$!+Kq--_CFA>iHBK(&|L=jR5;SfG@Cf4AJn8<4QhG)Rf>Q`)B|4auIB? zh7A{Ogg+2isUWPm9{a^mJ({%g2A+^a>adKk><~6ILzRN0O<%5!g{-e)_H0{v`}v!1_1$mYC6TV6WNXYb8zsBL@#|y&_MjpTaUY?Ptuj5C6fX5e_-e_KG>I?>N;~6r1 z=$3_X_nbZ*p5a_nbYyVwT2CObf@kicPKIrUxx_!4tH4&+eo-TDDD)$qQA0=aS^|Ht zjz1uGV|zuVcmeU6o?vhh&v>uK5VMbS-1$mWKlNkD=kwjg4X}eV8(RhkdIG^bZi73q zU1ptnLZKgWgXHQMDJtpl2g|#Cp_$FtiAMDV0{3wPMa_<(Ivj#(G0wLwq;9|e$GnWN zfg7w($B0(rJSUe2_X9YewkD9*>%GYn2>pW_uwBQn)bT*I7urB@zHCiVyw-5_@aG|v z#SPn~V{9t$v;|(`X}uq6IyfJ;!XW5l@CJiF^8|ydxxrx@88iDD;NA^9Eg6s|NE=SC ztq3a=3VpRZ82l}-ORL-&`JA&ljtUoNx&y(3a4)zg+#9FI76JMF{#&|2p?mo2SOjPB zS8_IYUIE9=Y2v_6q#+IrPyyg?C}+Sieom;6|7P=WPz6M&vcWa+eWF4C12a215GluW Q+5i9m07*qoM6N<$f?RLq=>Px# literal 0 HcmV?d00001 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt index 70dbd3b38..4df620443 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.latest.txt @@ -27,8 +27,9 @@ # supported version of external dependencies. # The instrumented model classes run inference in this process and the tests stub -# their runtimes, so no model backend extra is installed. -smolagents~=1.26.0 +# their runtimes. openai is the model backend the VCR agent test runs against; it +# is test-only, so pyproject.toml does not declare it. +smolagents[openai]~=1.26.0 wrapt>=2.2.2 -e util/opentelemetry-util-genai diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt index ef1f8d7db..29c25c5bb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/requirements.oldest.txt @@ -22,6 +22,10 @@ # come transitively from opentelemetry-test-util-genai. # # The instrumented model classes run inference in this process and the tests stub their -# runtimes, so there is no model backend to pin here. +# runtimes, so there is no backend to pin for them. openai is the model backend the VCR +# agent test instantiates. It is test-only, and neither pyproject.toml nor the +# instruments extra declares it. The pin is the version smolagents 1.24.0 declares in +# its own openai extra. +openai>=1.58.1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py new file mode 100644 index 000000000..7b9c3fc75 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_agents.py @@ -0,0 +1,849 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import gc +import inspect +import json +import pathlib +import tempfile +import weakref +from collections.abc import Generator +from types import GeneratorType, SimpleNamespace +from typing import Any + +import pytest +from smolagents import CodeAgent, OpenAIModel, ToolCallingAgent +from smolagents.models import ( + ChatMessage, + ChatMessageToolCall, + ChatMessageToolCallFunction, +) +from smolagents.monitoring import TokenUsage + +from opentelemetry.instrumentation.genai.smolagents import ( + patch as patch_module, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv.attributes import error_attributes +from opentelemetry.trace import StatusCode + +from .test_utils import ( + BrokenTool, + FakeCodeModel, + FakeStreamingCodeModel, + ImageTool, + ModelWithoutModelId, + NeverFinishingCodeModel, + attr, + data_point_attributes, + metrics_by_name, + parse_messages, + spans_by_operation, +) + + +def _operations(spans: list[Any]) -> list[str]: + return [ + (span.attributes or {}).get(GenAI.GEN_AI_OPERATION_NAME) + for span in spans + ] + + +def test_tool_calling_agent_with_image( + instrument_with_content, span_exporter, vcr +) -> None: + from PIL import Image + + model = OpenAIModel( + model_id="gpt-4o", + api_key="test_openai_api_key", + api_base="https://api.openai.com/v1", + ) + image_path = pathlib.Path(__file__).parent / "fixtures" / "img.png" + agent = ToolCallingAgent(tools=[], model=model, max_steps=3) + with vcr.use_cassette("agent_with_image.yaml"): + agent.run( + "Describe what you see in this image briefly.", + images=[Image.open(image_path)], + ) + + spans = span_exporter.get_finished_spans() + (agent_span,) = spans_by_operation(spans, "invoke_agent") + + # OpenAI SDK instrumentation is disabled, so only the agent emits a span. + assert sorted(filter(None, _operations(spans))) == ["invoke_agent"] + assert agent_span.name == "invoke_agent ToolCallingAgent" + + agent_inputs = parse_messages(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) + assert agent_inputs[0]["role"] == "user" + assert agent_inputs[0]["parts"][0] == { + "type": "text", + "content": "Describe what you see in this image briefly.", + } + assert agent_inputs[0]["parts"][1]["type"] == "blob" + assert agent_inputs[0]["parts"][1]["modality"] == "image" + + +def test_code_agent_non_streaming( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + result = agent.run("Test question") + assert result == "Test result from CodeAgent" + + spans = span_exporter.get_finished_spans() + (agent_span,) = spans_by_operation(spans, "invoke_agent") + + assert attr(agent_span, GenAI.GEN_AI_REQUEST_MODEL) == "fake-model" + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert json.loads(attr(agent_span, GenAI.GEN_AI_TOOL_DEFINITIONS)) == [ + { + "type": "function", + "name": "final_answer", + "description": "Provides a final answer to the given problem.", + "parameters": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The final answer to the problem", + } + }, + "required": ["answer"], + }, + } + ] + # Binding maps the positional value in ``run(task)`` to the ``task`` parameter. + inputs = parse_messages(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) + assert inputs[0] == { + "role": "user", + "parts": [{"type": "text", "content": "Test question"}], + } + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"][0]["content"] == "Test result from CodeAgent" + + +def test_code_agent_no_content(instrument_no_content, span_exporter) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + assert attr(agent_span, GenAI.GEN_AI_TOOL_DEFINITIONS) is not None + assert attr(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +def test_agent_event_only_records_no_inline_content( + instrument_event_only, span_exporter, log_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert attr(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert attr(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert not log_exporter.get_finished_logs() + + +def test_agent_span_and_event_records_content_on_span_only( + instrument_span_and_event, span_exporter, log_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + inputs = parse_messages(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert inputs[0]["parts"][0]["content"] == "Test question" + assert outputs[0]["parts"][0]["content"] == "Test result from CodeAgent" + assert not log_exporter.get_finished_logs() + + +def test_additional_args_reach_the_recorded_task( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question", additional_args={"city": "Paris"}) + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + (part,) = parse_messages(agent_span, GenAI.GEN_AI_INPUT_MESSAGES)[0][ + "parts" + ] + assert part["content"].startswith("Test question") + assert "city" in part["content"] + assert "Paris" in part["content"] + + +def test_run_returning_full_result_records_the_final_answer( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + result = agent.run("Test question", return_full_result=True) + assert result.output == "Test result from CodeAgent" + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"] == [ + {"type": "text", "content": "Test result from CodeAgent"} + ] + + +IMAGE_FINAL_ANSWER = """ +Thought: Return the image. +Code: +```py +final_answer(make_image()) +``` +""" + + +class _ImageAnswerModel(FakeCodeModel): + def generate(self, messages: list[Any], **kwargs: Any) -> ChatMessage: + return ChatMessage( + role="assistant", + content=IMAGE_FINAL_ANSWER, + token_usage=TokenUsage(input_tokens=1, output_tokens=1), + ) + + +def test_image_final_answer_is_recorded_as_a_blob( + instrument_with_content, span_exporter, monkeypatch +) -> None: + # Calling ``AgentImage.__str__`` writes a PNG and mutates the instance. + def _no_temp_dirs(*args: Any, **kwargs: Any) -> str: + raise AssertionError("telemetry wrote an image to disk") + + monkeypatch.setattr(tempfile, "mkdtemp", _no_temp_dirs) + + agent = CodeAgent( + tools=[ImageTool()], model=_ImageAnswerModel(), max_steps=2 + ) + agent.run("Make an image") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + (part,) = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES)[0][ + "parts" + ] + assert part["type"] == "blob" + assert part["modality"] == "image" + assert part["mime_type"] == "image/png" + + +def test_code_agent_streaming_is_lazy_until_drained( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + + assert ( + spans_by_operation(span_exporter.get_finished_spans(), "invoke_agent") + == [] + ) + + list(stream) + + spans = span_exporter.get_finished_spans() + (agent_span,) = spans_by_operation(spans, "invoke_agent") + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"][0]["content"] == "Test result from CodeAgent" + + +def test_streaming_run_stays_a_generator( + instrument_with_content, span_exporter +) -> None: + # Callers branch on these checks, so a wrapper that fails them changes + # behavior. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + + assert isinstance(stream, Generator) + assert inspect.isgenerator(stream) + assert stream.__class__ is GeneratorType + list(stream) + + +def test_streaming_run_records_no_chunk_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + # The chunk metrics describe the response stream of a generation call, + # and a chunk of an agent run is a step object. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + list(agent.run("Test question", stream=True)) + + metrics = metrics_by_name(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metrics + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK + not in metrics + ) + assert ( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK + not in metrics + ) + + +def test_streaming_run_with_stream_outputs( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent( + tools=[], + model=FakeStreamingCodeModel(), + max_steps=3, + stream_outputs=True, + ) + list(agent.run("Test question", stream=True)) + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"][0]["content"] == "Test result from CodeAgent" + + +def test_agent_run_metrics( + instrument_with_content, span_exporter, metric_reader +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + + metrics = metrics_by_name(metric_reader) + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + assert { + GenAI.GEN_AI_OPERATION_NAME: "invoke_agent", + GenAI.GEN_AI_REQUEST_MODEL: "fake-model", + } in data_point_attributes(duration) + # A run reports no token counts of its own: each model call records its + # own, and totalling the steps here would count them a second time. + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE not in metrics + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert attr(agent_span, GenAI.GEN_AI_USAGE_INPUT_TOKENS) is None + assert attr(agent_span, GenAI.GEN_AI_USAGE_OUTPUT_TOKENS) is None + + +@pytest.mark.parametrize("stream", [False, True]) +def test_run_that_exhausts_max_steps_is_not_reported_as_a_plain_stop( + instrument_with_content, span_exporter, stream: bool +) -> None: + # At ``max_steps``, smolagents records ``AgentMaxStepsError`` on the final + # ``ActionStep`` and returns normally. Reporting "stop" would hide that the + # agent reached the limit. + agent = CodeAgent(tools=[], model=NeverFinishingCodeModel(), max_steps=1) + if stream: + list(agent.run("Test question", stream=True)) + else: + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ( + "length", + ) + assert agent_span.status.status_code == StatusCode.UNSET + assert attr(agent_span, error_attributes.ERROR_TYPE) is None + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["finish_reason"] == "length" + + +def test_streaming_run_close_finalizes_once( + instrument_with_content, span_exporter +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + + # No ``FinalAnswerStep`` was observed, so no output is recorded. + stream.close() + stream.close() + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert len(agent_spans) == 1 + assert attr(agent_spans[0], GenAI.GEN_AI_OUTPUT_MESSAGES) is None + + +@pytest.mark.parametrize("use_context_manager", [True, False]) +def test_streaming_run_stopped_midway_preserves_close_error( + instrument_with_content, span_exporter, use_context_manager: bool +) -> None: + # smolagents yields from a finally block, so its generator rejects an early + # close. Instrumentation must preserve that exception. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + stream = agent.run("Test question", stream=True) + with pytest.raises(RuntimeError, match="generator ignored GeneratorExit"): + if use_context_manager: + with stream: + for _ in stream: + break + else: + for _ in stream: + break + stream.close() + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.ERROR + assert attr(agent_span, error_attributes.ERROR_TYPE) == "RuntimeError" + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None + + +def test_streaming_run_close_reraises_a_real_cleanup_error( + instrument_with_content, span_exporter, lifecycle, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _failing_cleanup(*args: Any, **kwargs: Any): + try: + yield SimpleNamespace(token_usage=None) + finally: + raise RuntimeError("cleanup exploded") + + monkeypatch.setattr(agent, "_run_stream", _failing_cleanup) + stream = agent.run("Test question", stream=True) + for _ in stream: + break + with pytest.raises(RuntimeError, match="cleanup exploded"): + stream.close() + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.ERROR + assert attr(agent_span, error_attributes.ERROR_TYPE) == "RuntimeError" + assert lifecycle.leaked == [] + + +def test_caller_error_inside_stream_context( + instrument_with_content, span_exporter, lifecycle +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + with pytest.raises(RuntimeError, match="caller exploded"): + with agent.run("Test question", stream=True) as stream: + next(stream) + raise RuntimeError("caller exploded") + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert len(agent_spans) == 1 + assert agent_spans[0].status.status_code == StatusCode.ERROR + assert attr(agent_spans[0], error_attributes.ERROR_TYPE) == "RuntimeError" + assert lifecycle.leaked == [] + + +def test_run_bad_call_records_an_error_span( + instrument_with_content, span_exporter, lifecycle +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + with pytest.raises(TypeError): + agent.run() + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.ERROR + assert attr(agent_span, error_attributes.ERROR_TYPE) == "TypeError" + assert attr(agent_span, GenAI.GEN_AI_INPUT_MESSAGES) is None + assert lifecycle.leaked == [] + + +def test_run_failure_before_stream( + instrument_with_content, span_exporter, lifecycle, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _boom(*args: Any, **kwargs: Any): + raise RuntimeError("boom") + + monkeypatch.setattr(agent, "_run_stream", _boom) + with pytest.raises(RuntimeError, match="boom"): + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.ERROR + assert attr(agent_span, error_attributes.ERROR_TYPE) == "RuntimeError" + assert lifecycle.leaked == [] + + +def test_stream_failure_during_iteration( + instrument_with_content, span_exporter, lifecycle, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _bad_stream(*args: Any, **kwargs: Any): + yield SimpleNamespace(token_usage=None) + raise ConnectionError("stream died") + + monkeypatch.setattr(agent, "_run_stream", _bad_stream) + stream = agent.run("Test question", stream=True) + with pytest.raises(ConnectionError, match="stream died"): + list(stream) + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert len(agent_spans) == 1 + assert agent_spans[0].status.status_code == StatusCode.ERROR + assert ( + attr(agent_spans[0], error_attributes.ERROR_TYPE) == "ConnectionError" + ) + assert lifecycle.leaked == [] + + +def test_interrupted_stream_iteration_ends_the_span( + instrument_with_content, span_exporter, lifecycle, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _interrupt(*args: Any, **kwargs: Any): + if False: + yield None + raise KeyboardInterrupt + + monkeypatch.setattr(agent, "_run_stream", _interrupt) + stream = agent.run("Test question", stream=True) + + with pytest.raises(KeyboardInterrupt): + next(stream) + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.UNSET + assert lifecycle.leaked == [] + + +def test_interrupted_agent_recording_ends_the_span( + instrument_with_content, span_exporter, lifecycle, monkeypatch +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _interrupt(*args: Any, **kwargs: Any) -> Any: + raise KeyboardInterrupt + + monkeypatch.setattr(patch_module, "to_tool_definitions", _interrupt) + + with pytest.raises(KeyboardInterrupt): + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.UNSET + assert lifecycle.leaked == [] + + +@pytest.fixture +def signature_calls(monkeypatch) -> list[str]: + original_signature = patch_module.signature + calls: list[str] = [] + + def _signature(callable_: Any): + calls.append(getattr(callable_, "__qualname__", repr(callable_))) + return original_signature(callable_) + + monkeypatch.setattr(patch_module, "_signatures", {}) + monkeypatch.setattr(patch_module, "signature", _signature) + return calls + + +def test_agent_run_caches_the_signature( + instrument_with_content, signature_calls: list[str] +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + agent.run("First question") + agent.run("Second question") + + assert signature_calls == ["MultiStepAgent.run"] + + +def test_the_signature_cache_does_not_grow_per_agent( + instrument_with_content, signature_calls: list[str] +) -> None: + agents = [ + CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + for _ in range(5) + ] + for index, agent in enumerate(agents): + agent.run(f"Question {index}") + + assert signature_calls == ["MultiStepAgent.run"] + assert len(patch_module._signatures) == 1 + + +def test_the_signature_cache_does_not_retain_agents( + instrument_with_content, signature_calls: list[str] +) -> None: + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + agent.run("Test question") + reference = weakref.ref(agent) + + del agent + gc.collect() + + assert reference() is None + + +def test_interrupted_run_ends_the_span( + instrument_with_content, span_exporter, lifecycle, monkeypatch +) -> None: + # A KeyboardInterrupt is not an Exception. Catching only Exception would + # leave the span of an interrupted run open for the process's lifetime. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + + def _interrupt(*args: Any, **kwargs: Any): + raise KeyboardInterrupt + + monkeypatch.setattr(agent, "_run_stream", _interrupt) + with pytest.raises(KeyboardInterrupt): + agent.run("Test question") + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + # Interrupting a run is not the agent failing, so the span ends without an + # error. + assert agent_span.status.status_code == StatusCode.UNSET + assert lifecycle.leaked == [] + + +@pytest.mark.parametrize("stream", [False, True]) +def test_a_failed_recording_does_not_break_a_finished_run( + instrument_with_content, + span_exporter, + lifecycle, + monkeypatch, + stream: bool, +) -> None: + # A recording failure must not replace the stream's ``StopIteration``. + def raise_error(*args: Any, **kwargs: Any) -> Any: + raise ValueError("unexpected answer shape") + + monkeypatch.setattr(patch_module, "final_answer_parts", raise_error) + + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + if stream: + assert list(agent.run("Test question", stream=True)) + else: + assert agent.run("Test question") == "Test result from CodeAgent" + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.UNSET + assert attr(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) is None + assert lifecycle.leaked == [] + + +def test_a_failed_recording_before_the_run_does_not_break_it( + instrument_with_content, span_exporter, lifecycle, monkeypatch +) -> None: + def raise_error(*args: Any, **kwargs: Any) -> Any: + raise ValueError("unexpected tool shape") + + monkeypatch.setattr(patch_module, "to_tool_definitions", raise_error) + + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + assert agent.run("Test question") == "Test result from CodeAgent" + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.UNSET + assert attr(agent_span, GenAI.GEN_AI_TOOL_DEFINITIONS) is None + assert lifecycle.leaked == [] + + +def test_a_model_without_a_model_id_is_supported( + instrument_with_content, span_exporter, lifecycle +) -> None: + # smolagents allows models without ``model_id``. Instrumentation must also + # allow them. + agent = CodeAgent(tools=[], model=ModelWithoutModelId(), max_steps=3) + assert agent.run("Test question") == "Test result from CodeAgent" + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert attr(agent_span, GenAI.GEN_AI_REQUEST_MODEL) is None + assert lifecycle.leaked == [] + + +class _ManagerModel: + model_id = "manager-model" + kwargs: dict[str, Any] = {} + + def generate(self, messages, **kwargs) -> ChatMessage: + # The first step starts with three messages. + if len(messages) < 4: + content = ( + "Thought: delegate.\nCode:\n```py\n" + 'search_agent("Who is the president?")\n```' + ) + else: + content = ( + "Thought: finish.\nCode:\n```py\n" + 'final_answer("Final report.")\n```' + ) + return ChatMessage( + role="assistant", + content=content, + token_usage=TokenUsage(input_tokens=5, output_tokens=3), + ) + + def __call__(self, *args, **kwargs) -> ChatMessage: + return self.generate(*args, **kwargs) + + +class _ManagedModel: + model_id = "managed-model" + kwargs: dict[str, Any] = {} + + def generate(self, messages, **kwargs) -> ChatMessage: + return ChatMessage( + role="assistant", + content="", + tool_calls=[ + ChatMessageToolCall( + id="call_0", + type="function", + function=ChatMessageToolCallFunction( + name="final_answer", + arguments="Report on the president", + ), + ) + ], + token_usage=TokenUsage(input_tokens=4, output_tokens=2), + ) + + def __call__(self, *args, **kwargs) -> ChatMessage: + return self.generate(*args, **kwargs) + + def parse_tool_calls(self, message: ChatMessage) -> ChatMessage: + return message + + +def test_managed_agent_run_is_recorded( + instrument_with_content, span_exporter +) -> None: + managed = ToolCallingAgent( + tools=[], + model=_ManagedModel(), + max_steps=3, + name="search_agent", + description="Runs searches.", + ) + manager = CodeAgent( + tools=[], + model=_ManagerModel(), + managed_agents=[managed], + max_steps=4, + ) + assert manager.run("Fake question.") == "Final report." + + agent_spans = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + by_name = {span.name: span for span in agent_spans} + assert "invoke_agent search_agent" in by_name + manager_span = next( + span for name, span in by_name.items() if "search_agent" not in name + ) + managed_span = by_name["invoke_agent search_agent"] + # ``CodeAgent`` runs managed agents in a worker thread without caller + # context. The managed run therefore starts a separate trace. + assert managed_span.parent is None + assert ( + attr(managed_span, GenAI.GEN_AI_AGENT_DESCRIPTION) == "Runs searches." + ) + assert attr(manager_span, GenAI.GEN_AI_AGENT_DESCRIPTION) is None + # Managed agents are exposed to the manager model as tools. + definitions = json.loads(attr(manager_span, GenAI.GEN_AI_TOOL_DEFINITIONS)) + assert [definition["name"] for definition in definitions] == [ + "final_answer", + "search_agent", + ] + search_agent = definitions[1] + assert search_agent["description"] == "Runs searches." + assert "task" in search_agent["parameters"]["properties"] + + +class _RecoveringModel: + model_id = "recovering-model" + kwargs: dict[str, Any] = {} + + def __init__(self) -> None: + self.calls = 0 + + def generate(self, messages, **kwargs) -> ChatMessage: + self.calls += 1 + if self.calls == 1: + name, arguments = "broken_tool", {"location": "Paris"} + else: + name, arguments = "final_answer", "recovered" + return ChatMessage( + role="assistant", + content="", + tool_calls=[ + ChatMessageToolCall( + id=f"call_{self.calls}", + type="function", + function=ChatMessageToolCallFunction( + name=name, arguments=arguments + ), + ) + ], + token_usage=TokenUsage(input_tokens=3, output_tokens=1), + ) + + def __call__(self, *args, **kwargs) -> ChatMessage: + return self.generate(*args, **kwargs) + + def parse_tool_calls(self, message: ChatMessage) -> ChatMessage: + return message + + +def test_a_tool_error_the_agent_recovers_from_leaves_the_run_successful( + instrument_with_content, span_exporter +) -> None: + # smolagents gives tool errors back to the model, so a recovered run stays + # successful. + agent = ToolCallingAgent( + tools=[BrokenTool()], model=_RecoveringModel(), max_steps=4 + ) + assert agent.run("Do the thing") == "recovered" + + (agent_span,) = spans_by_operation( + span_exporter.get_finished_spans(), "invoke_agent" + ) + assert agent_span.status.status_code == StatusCode.UNSET + assert attr(agent_span, error_attributes.ERROR_TYPE) is None + assert attr(agent_span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) == ("stop",) + outputs = parse_messages(agent_span, GenAI.GEN_AI_OUTPUT_MESSAGES) + assert outputs[0]["parts"][0]["content"] == "recovered" diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py index dc0ba1e7c..3a68aa217 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_conformance.py @@ -20,6 +20,7 @@ run_conformance, ) +from .conformance.agent import AgentScenario from .conformance.inference import ( ChatScenario, StreamedChatScenario, @@ -31,6 +32,7 @@ @pytest.mark.parametrize( "scenario", [ + pytest.param(AgentScenario()), pytest.param(ChatScenario()), pytest.param(StreamedChatScenario()), pytest.param(ToolDefinitionsScenario()), diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py index fcdaad6c3..4efa0d4ae 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_instrumentor.py @@ -10,17 +10,19 @@ import pytest import smolagents +from smolagents import CodeAgent from wrapt import wrap_function_wrapper from opentelemetry.instrumentation.genai.smolagents import ( SmolagentsInstrumentor, _model_classes_defining, ) +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.util._importlib_metadata import entry_points from opentelemetry.util.genai.completion_hook import CompletionHook -from .test_utils import MESSAGES, transformers_model +from .test_utils import MESSAGES, FakeCodeModel, transformers_model class RecordingHook(CompletionHook): @@ -31,10 +33,24 @@ def on_completion(self, **kwargs: Any) -> None: self.calls.append(kwargs) +class RaisingHook(CompletionHook): + def on_completion(self, **kwargs: Any) -> None: + raise RuntimeError("hook failed") + + def _generate_a_chat_span() -> None: transformers_model().generate(messages=MESSAGES) +def _passthrough_wrapper( + wrapped: Any, + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Any: + return wrapped(*args, **kwargs) + + def test_entrypoint_loads_instrumentor() -> None: (entrypoint,) = entry_points( group="opentelemetry_instrumentor", name="smolagents" @@ -54,6 +70,7 @@ def test_instrument_uninstrument_restores_originals( original_generate = smolagents.TransformersModel.generate original_mlx_generate = smolagents.MLXModel.generate original_generate_stream = smolagents.TransformersModel.generate_stream + original_run = smolagents.MultiStepAgent.run instrumentor = SmolagentsInstrumentor() instrumentor.instrument( @@ -68,6 +85,7 @@ def test_instrument_uninstrument_restores_originals( smolagents.TransformersModel.generate_stream is not original_generate_stream ) + assert smolagents.MultiStepAgent.run is not original_run instrumentor.uninstrument() @@ -77,6 +95,7 @@ def test_instrument_uninstrument_restores_originals( smolagents.TransformersModel.generate_stream is original_generate_stream ) + assert smolagents.MultiStepAgent.run is original_run @pytest.mark.parametrize( @@ -122,6 +141,7 @@ def test_uninstrument_through_a_new_constructor_call( # .uninstrument() form must restore everything even though the second # constructor call re-runs __init__ on the live instance. original_generate = smolagents.TransformersModel.generate + original_run = smolagents.MultiStepAgent.run SmolagentsInstrumentor().instrument( tracer_provider=tracer_provider, @@ -131,6 +151,7 @@ def test_uninstrument_through_a_new_constructor_call( SmolagentsInstrumentor().uninstrument() assert smolagents.TransformersModel.generate is original_generate + assert smolagents.MultiStepAgent.run is original_run @pytest.mark.parametrize("method", ["generate", "generate_stream"]) @@ -163,6 +184,7 @@ def test_repeated_instrument_uninstrument( # BaseInstrumentor returns a per-class singleton, so the wrapped-class # bookkeeping has to survive being filled and drained more than once. original_generate = smolagents.TransformersModel.generate + original_run = smolagents.MultiStepAgent.run instrumentor = SmolagentsInstrumentor() for _ in range(2): @@ -172,8 +194,10 @@ def test_repeated_instrument_uninstrument( meter_provider=meter_provider, ) assert smolagents.TransformersModel.generate is not original_generate + assert smolagents.MultiStepAgent.run is not original_run instrumentor.uninstrument() assert smolagents.TransformersModel.generate is original_generate + assert smolagents.MultiStepAgent.run is original_run def test_uninstrument_without_instrument() -> None: @@ -181,60 +205,78 @@ def test_uninstrument_without_instrument() -> None: # also be a no-op on unpatched attributes: the rollback in _instrument() # calls it after a partial patch. original_generate = smolagents.TransformersModel.generate + original_run = smolagents.MultiStepAgent.run SmolagentsInstrumentor().uninstrument() SmolagentsInstrumentor()._uninstrument() assert smolagents.TransformersModel.generate is original_generate + assert smolagents.MultiStepAgent.run is original_run def test_instrument_with_no_providers() -> None: # Without providers the handler falls back to the globals; instrumenting # must not require a caller to pass them. original_generate = smolagents.TransformersModel.generate + original_run = smolagents.MultiStepAgent.run instrumentor = SmolagentsInstrumentor() instrumentor.instrument() try: assert smolagents.TransformersModel.generate is not original_generate + assert smolagents.MultiStepAgent.run is not original_run finally: instrumentor.uninstrument() assert smolagents.TransformersModel.generate is original_generate + assert smolagents.MultiStepAgent.run is original_run + + +def _originals() -> dict[tuple[type, str], Any]: + patched: dict[tuple[type, str], Any] = { + (model_cls, "generate"): model_cls.__dict__["generate"] + for model_cls in _model_classes_defining("generate") + } + patched.update( + { + (model_cls, "generate_stream"): model_cls.__dict__[ + "generate_stream" + ] + for model_cls in _model_classes_defining("generate_stream") + } + ) + patched[(smolagents.MultiStepAgent, "run")] = ( + smolagents.MultiStepAgent.__dict__["run"] + ) + return patched +@pytest.mark.parametrize("fail_on_call", [2, None]) def test_failed_instrument_rolls_back_partial_patches( - tracer_provider, logger_provider, meter_provider + tracer_provider, logger_provider, meter_provider, fail_on_call: int | None ) -> None: - # A failure part-way through must leave no class patched, because - # uninstrument() cannot clean up after a failed _instrument(). - model_classes = _model_classes_defining("generate") - assert len(model_classes) > 1, ( - "the rollback needs more than one class to patch" - ) - originals = { - model_cls: model_cls.__dict__["generate"] - for model_cls in model_classes - } - stream_classes = _model_classes_defining("generate_stream") - stream_originals = { - model_cls: model_cls.__dict__["generate_stream"] - for model_cls in stream_classes - } + # A failure part-way through must leave nothing patched because + # uninstrument() cannot clean up after a failed _instrument(). A value of 2 + # fails after the first patch. None fails on MultiStepAgent.run, the last + # patch. + originals = _originals() + assert len(originals) > 1, "the rollback needs more than one patch" real_wrap = wrap_function_wrapper calls = 0 - def fail_on_the_second_class(target: Any, name: str, wrapper: Any) -> None: + def failing_wrap(target: Any, name: str, wrapper: Any) -> None: nonlocal calls calls += 1 - if calls == 2: + if calls == fail_on_call or ( + fail_on_call is None and name == "MultiStepAgent.run" + ): raise RuntimeError("boom") real_wrap(target, name, wrapper) with patch( "opentelemetry.instrumentation.genai.smolagents.wrap_function_wrapper", - fail_on_the_second_class, + failing_wrap, ): with pytest.raises(RuntimeError, match="boom"): SmolagentsInstrumentor().instrument( @@ -243,11 +285,89 @@ def fail_on_the_second_class(target: Any, name: str, wrapper: Any) -> None: meter_provider=meter_provider, ) - assert calls == 2, "the first class was expected to be patched" - for model_cls, original in originals.items(): - assert model_cls.__dict__["generate"] is original - for model_cls, original in stream_originals.items(): - assert model_cls.__dict__["generate_stream"] is original + assert calls > 1, "at least one attribute was expected to be patched" + for (target, name), original in originals.items(): + assert target.__dict__[name] is original, ( + f"{target.__name__}.{name} was not restored" + ) + + +@pytest.mark.parametrize("fail_on_call", [2, None]) +def test_failed_instrument_preserves_third_party_wrappers( + tracer_provider, logger_provider, meter_provider, fail_on_call: int | None +) -> None: + original_run = smolagents.MultiStepAgent.__dict__["run"] + + wrap_function_wrapper( + smolagents.MultiStepAgent, "run", _passthrough_wrapper + ) + third_party_run = smolagents.MultiStepAgent.__dict__["run"] + + real_wrap = wrap_function_wrapper + calls = 0 + + def failing_wrap(target: Any, name: str, wrapper: Any) -> None: + nonlocal calls + calls += 1 + if calls == fail_on_call or ( + fail_on_call is None and name == "MultiStepAgent.run" + ): + raise RuntimeError("boom") + real_wrap(target, name, wrapper) + + try: + with patch( + "opentelemetry.instrumentation.genai.smolagents.wrap_function_wrapper", + failing_wrap, + ): + with pytest.raises(RuntimeError, match="boom"): + SmolagentsInstrumentor().instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + + assert smolagents.MultiStepAgent.__dict__["run"] is third_party_run + finally: + unwrap(smolagents.MultiStepAgent, "run") + + assert smolagents.MultiStepAgent.__dict__["run"] is original_run + + +def test_uninstrument_keeps_third_party_wrappers_working( + tracer_provider, logger_provider, meter_provider, span_exporter +) -> None: + original_run = smolagents.MultiStepAgent.__dict__["run"] + calls: list[str] = [] + + def recording_wrapper( + wrapped: Any, + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + calls.append("third party") + return wrapped(*args, **kwargs) + + wrap_function_wrapper(smolagents.MultiStepAgent, "run", recording_wrapper) + + try: + instrumentor = SmolagentsInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + instrumentor.uninstrument() + + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + assert agent.run("Test question") == "Test result from CodeAgent" + assert calls == ["third party"] + assert span_exporter.get_finished_spans() == () + finally: + unwrap(smolagents.MultiStepAgent, "run") + + assert smolagents.MultiStepAgent.__dict__["run"] is original_run def test_a_user_subclass_inherits_the_patched_generate( @@ -299,6 +419,43 @@ def test_explicit_completion_hook_takes_precedence( assert explicit_hook.calls, "explicit completion hook was not invoked" +@pytest.mark.parametrize("agent_fails", [False, True]) +def test_completion_hook_failure_reaches_the_caller( + tracer_provider, + logger_provider, + meter_provider, + span_exporter, + lifecycle, + monkeypatch, + agent_fails: bool, +) -> None: + # ``load_completion_hook`` wraps environment hooks so their errors do not + # reach the caller. + agent = CodeAgent(tools=[], model=FakeCodeModel(), max_steps=3) + if agent_fails: + + def _fail(*args: Any, **kwargs: Any) -> Any: + raise ValueError("agent failed") + + monkeypatch.setattr(agent, "_run_stream", _fail) + + with instrument( + SmolagentsInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + completion_hook=RaisingHook(), + ): + with pytest.raises(RuntimeError, match="hook failed") as caught: + agent.run("Test question") + + if agent_fails: + assert isinstance(caught.value.__context__, ValueError) + assert len(span_exporter.get_finished_spans()) == 1 + assert lifecycle.leaked == [] + + def test_env_completion_hook_used_when_no_explicit_hook( tracer_provider, logger_provider, meter_provider ) -> None: diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py index 196d39979..48ef42d4c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_models.py @@ -20,7 +20,6 @@ import pytest from smolagents.models import ChatMessage, MessageRole -from opentelemetry.context import Context from opentelemetry.instrumentation.genai.smolagents import ( patch as patch_module, ) @@ -36,7 +35,6 @@ from opentelemetry.instrumentation.genai.smolagents.provider import ( resolve_provider, ) -from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, ) @@ -202,37 +200,6 @@ def test_runtime_error_is_recorded_and_reraised( assert attr(span, GenAI.GEN_AI_RESPONSE_FINISH_REASONS) is None -class _LifecycleRecorder(SpanProcessor): - """Records span starts and ends. - - The exporter only sees a span once it ends, so an unfinished span reads - there as no span at all. - """ - - def __init__(self) -> None: - self.started: list[str] = [] - self.ended: list[str] = [] - - def on_start( - self, span: Span, parent_context: Context | None = None - ) -> None: - self.started.append(span.name) - - def on_end(self, span: ReadableSpan) -> None: - self.ended.append(span.name) - - @property - def leaked(self) -> list[str]: - return self.started[len(self.ended) :] - - -@pytest.fixture -def lifecycle(tracer_provider) -> _LifecycleRecorder: - recorder = _LifecycleRecorder() - tracer_provider.add_span_processor(recorder) - return recorder - - def test_lifecycle_recorder_sees_the_happy_path( instrument_with_content, lifecycle ) -> None: diff --git a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py index efec5fadb..c668f38ae 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-smolagents/tests/test_utils.py @@ -3,12 +3,12 @@ """Shared helpers, tools, and model stubs for smolagents instrumentation tests. -Only the in-process model classes are instrumented, and none of them can be -recorded with VCR: they run inference in the current process instead of calling -a provider over HTTP. Each factory here bypasses ``__init__`` (which would load -gigabytes of weights) and stubs the runtime pieces the real ``generate`` drives, -so the code under test is smolagents' own ``generate`` and the wrapper around -it. +The in-process model helpers cannot be recorded with VCR because they make no +HTTP requests. Each factory bypasses ``__init__`` to avoid loading model weights +and stubs the runtime code that the real ``generate`` method calls. The tests +therefore exercise smolagents' ``generate`` method and its wrapper. + +Agent fakes emit no ``chat`` spans. """ from __future__ import annotations @@ -19,13 +19,110 @@ from typing import Any import pytest +from PIL import Image from smolagents import Tool +from smolagents.models import ChatMessage, ChatMessageStreamDelta +from smolagents.monitoring import TokenUsage -from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.context import Context +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAI, ) +CODE_FINAL_ANSWER = """ +Thought: Return the final answer. +Code: +```py +final_answer("Test result from CodeAgent") +``` +""" + +CODE_NO_OP = """ +Thought: Keep going. +Code: +```py +x = 1 +``` +""" + + +class FakeCodeModel: + """Drive a CodeAgent without emitting a ``chat`` span.""" + + def __init__(self, model_id: str = "fake-model") -> None: + self.model_id = model_id + self.kwargs: dict[str, Any] = {} + + def generate( + self, + messages: list[Any], + stop_sequences: list[str] | None = None, + response_format: Any = None, + tools_to_call_from: list[Any] | None = None, + **kwargs: Any, + ) -> ChatMessage: + return ChatMessage( + role="assistant", + content=CODE_FINAL_ANSWER, + token_usage=TokenUsage(input_tokens=11, output_tokens=7), + ) + + def __call__(self, *args: Any, **kwargs: Any) -> ChatMessage: + return self.generate(*args, **kwargs) + + +class ModelWithoutModelId(FakeCodeModel): + def __init__(self) -> None: + super().__init__() + del self.model_id + + +class FakeStreamingCodeModel(FakeCodeModel): + def generate_stream( + self, + messages: list[Any], + stop_sequences: list[str] | None = None, + response_format: Any = None, + tools_to_call_from: list[Any] | None = None, + **kwargs: Any, + ) -> Any: + yield ChatMessageStreamDelta( + content=CODE_FINAL_ANSWER, + token_usage=TokenUsage(input_tokens=11, output_tokens=7), + ) + + +class NeverFinishingCodeModel(FakeCodeModel): + """Trigger smolagents' synthesized max-step answer.""" + + def generate(self, messages: list[Any], **kwargs: Any) -> ChatMessage: + return ChatMessage( + role="assistant", + content=CODE_NO_OP, + token_usage=TokenUsage(input_tokens=3, output_tokens=5), + ) + + +class ImageTool(Tool): + name = "make_image" + description = "Make a tiny image" + inputs: dict[str, Any] = {} + output_type = "image" + + def forward(self) -> Any: + return Image.new("RGB", (4, 4), color="red") + + +class BrokenTool(Tool): + name = "broken_tool" + description = "A tool that always fails" + inputs = {"location": {"type": "string", "description": "ignored"}} + output_type = "string" + + def forward(self, location: str) -> str: + raise ValueError("tool exploded") + class GetWeatherTool(Tool): name = "get_weather" @@ -229,3 +326,27 @@ def metrics_by_name(metric_reader: Any) -> dict[str, Any]: def data_point_attributes(metric: Any) -> list[dict[str, Any]]: return [dict(point.attributes) for point in metric.data.data_points] + + +class LifecycleRecorder(SpanProcessor): + """Records span starts and ends. + + The exporter only sees a span once it ends, so an unfinished span reads + there as no span at all. + """ + + def __init__(self) -> None: + self.started: list[str] = [] + self.ended: list[str] = [] + + def on_start( + self, span: Span, parent_context: Context | None = None + ) -> None: + self.started.append(span.name) + + def on_end(self, span: ReadableSpan) -> None: + self.ended.append(span.name) + + @property + def leaked(self) -> list[str]: + return self.started[len(self.ended) :]