[opentelemetry-instrumentation-genai-smolagents] instrument agent runs

instrumentation/opentelemetry-instrumentation-genai-smolagents/.changelog/403.added

@@ -0,0 +1 @@
+Add ``invoke_agent`` spans for streaming and non-streaming ``MultiStepAgent.run()`` calls.

instrumentation/opentelemetry-instrumentation-genai-smolagents/README.rst

@@ -36,10 +36,6 @@ for the client library instead:
      - the instrumentation or built-in telemetry of the client library the model
        calls (``huggingface_hub``, ``litellm``)
 
-Agent runs (``invoke_agent``) and tool calls (``execute_tool``) are not
-instrumented yet. A model call made inside an agent run still gets a ``chat``
-span, but no agent span sits above it.
-
 ``TransformersModel`` is the only instrumented class with a ``generate_stream``.
 A streamed call gets a ``chat`` span that stays open until the caller drains the
 deltas. This covers both ``stream_outputs=True`` on an agent and a direct
@@ -50,13 +46,18 @@ call also records the
 
 Known gaps:
 
+* A managed agent called from the generated code of a ``CodeAgent`` runs in a
+  worker thread of the local Python executor. That worker does not carry the
+  calling context, so the managed agent's ``invoke_agent`` span starts a trace
+  of its own instead of nesting under the manager's span.
 * A subclass that inherits ``generate`` or ``generate_stream`` from one of the
   three classes above is instrumented. A subclass that overrides one is not: the
   override shadows the patched method, so the call produces no ``chat`` span.
 * A ``chat`` span reports no ``gen_ai.response.id``, no
   ``gen_ai.response.model``, no ``gen_ai.response.finish_reasons`` and no
   ``server.address``. A runtime in this process returns the generated text and
   the token counts, nothing more. It also listens on no socket.
+* Tool calls (``execute_tool``) are not instrumented yet.
 
 Installation
 ------------
@@ -73,21 +74,13 @@ Usage
     from opentelemetry.instrumentation.genai.smolagents import (
         SmolagentsInstrumentor,
     )
-    from smolagents import TransformersModel
+    from smolagents import CodeAgent, TransformersModel
 
     SmolagentsInstrumentor().instrument()
 
     model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct")
-    model.generate(
-        [
-            {
-                "role": "user",
-                "content": [
-                    {"type": "text", "text": "How many seconds are in a week?"}
-                ],
-            }
-        ]
-    )
+    agent = CodeAgent(tools=[], model=model)
+    agent.run("How many seconds are in a week?")
 
 Configuration
 -------------

instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/__init__.py

@@ -11,8 +11,8 @@
 ``MLXModel``) are recorded as ``chat`` spans. The API-backed model classes are
 not instrumented here: each one calls a client library that carries its own
 instrumentation, and emitting a span at this layer as well would duplicate the
-span and count the token-usage and duration metrics twice. Agent runs and tool
-calls are not instrumented yet.
+span and count the token-usage and duration metrics twice. Tool calls are not
+instrumented yet.
 
 Usage
 -----
@@ -22,21 +22,13 @@
     from opentelemetry.instrumentation.genai.smolagents import (
         SmolagentsInstrumentor,
     )
-    from smolagents import TransformersModel
+    from smolagents import CodeAgent, TransformersModel
 
     SmolagentsInstrumentor().instrument()
 
     model = TransformersModel(model_id="HuggingFaceTB/SmolLM2-135M-Instruct")
-    model.generate(
-        [
-            {
-                "role": "user",
-                "content": [
-                    {"type": "text", "text": "How many seconds are in a week?"}
-                ],
-            }
-        ]
-    )
+    agent = CodeAgent(tools=[], model=model)
+    agent.run("How many seconds are in a week?")
 
 Configuration
 -------------
@@ -60,6 +52,7 @@
 from collections.abc import Collection
 from typing import Any
 
+import smolagents
 from smolagents import models
 from wrapt import wrap_function_wrapper
 
@@ -69,11 +62,14 @@
 from opentelemetry.util.genai.handler import TelemetryHandler
 
 from .package import _instruments
-from .patch import model_generate, model_generate_stream
+from .patch import (
+    agent_run,
+    model_generate,
+    model_generate_stream,
+)
 
 __all__ = ["SmolagentsInstrumentor"]
 
-
 # The model classes that run inference in the current process. They call no
 # client library, so this instrumentation is the only place their model calls
 # can be observed.
@@ -120,6 +116,7 @@ class SmolagentsInstrumentor(BaseInstrumentor):
     # only ``_instrument`` / ``_uninstrument`` rebind.
     _wrapped_generate_classes: list[type] = []
     _wrapped_generate_stream_classes: list[type] = []
+    _wrapped_agent_run = False
 
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
@@ -144,6 +141,7 @@ def _instrument(self, **kwargs: Any) -> None:
 
         self._wrapped_generate_classes = []
         self._wrapped_generate_stream_classes = []
+        self._wrapped_agent_run = False
         try:
             for model_cls in _model_classes_defining("generate"):
                 wrap_function_wrapper(
@@ -160,6 +158,16 @@ def _instrument(self, **kwargs: Any) -> None:
                     model_generate_stream(handler),
                 )
                 self._wrapped_generate_stream_classes.append(model_cls)
+
+            # TODO: emit a span per agent step once the semantic conventions
+            # define an operation for one iteration of a reason-and-act loop.
+            # https://github.com/open-telemetry/semantic-conventions-genai/issues/81
+            wrap_function_wrapper(
+                "smolagents",
+                "MultiStepAgent.run",
+                agent_run(handler),
+            )
+            self._wrapped_agent_run = True
         except BaseException:
             # BaseInstrumentor.instrument() doesn't mark the instrumentor as
             # instrumented when _instrument raises, so uninstrument() would
@@ -176,3 +184,7 @@ def _uninstrument(self, **kwargs: Any) -> None:
         for model_cls in self._wrapped_generate_stream_classes:
             unwrap(model_cls, "generate_stream")
         self._wrapped_generate_stream_classes = []
+
+        if self._wrapped_agent_run:
+            unwrap(smolagents.MultiStepAgent, "run")
+            self._wrapped_agent_run = False

instrumentation/opentelemetry-instrumentation-genai-smolagents/src/opentelemetry/instrumentation/genai/smolagents/_messages.py

@@ -1,11 +1,12 @@
 # Copyright The OpenTelemetry Authors
 # SPDX-License-Identifier: Apache-2.0
 
-"""Convert smolagents message and tool shapes into util-genai GenAI types.
+"""Convert smolagents message, tool, and agent-run values to GenAI types.
 
 smolagents passes ``generate(messages=...)`` a list of ``ChatMessage`` objects
-or plain dicts, and returns a ``ChatMessage``. This module maps those, and the
-``tools_to_call_from`` tool objects, onto the types in
+or plain dicts, and returns a ``ChatMessage``. ``MultiStepAgent.run`` takes a
+task string and optional images, and returns a final answer. This module maps
+those values, and the ``tools_to_call_from`` tool objects, onto the types in
 ``opentelemetry.util.genai.types``. util-genai then serializes them into
 ``gen_ai.input.messages``, ``gen_ai.output.messages``, and
 ``gen_ai.tool.definitions``.
@@ -16,9 +17,11 @@
 import base64
 import binascii
 import logging
+from collections.abc import Sequence
 from enum import Enum
-from typing import TYPE_CHECKING, Any, TypeAlias
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 
+from PIL.Image import Image
 from smolagents.models import get_tool_json_schema
 from smolagents.utils import encode_image_base64
 
@@ -34,7 +37,7 @@
 )
 
 if TYPE_CHECKING:
-    from PIL.Image import Image
+    from smolagents.agents import MultiStepAgent
     from smolagents.models import ChatMessage, MessageRole
     from smolagents.tools import Tool
 
@@ -46,6 +49,11 @@
 # ``image_url``, and a PIL image under ``image``.
 _ContentElement: TypeAlias = dict[str, Any]
 
+# Something the model can call. A managed agent is one too: the manager sets the
+# agent's ``inputs`` and ``output_type`` and passes it in ``tools_to_call_from``
+# alongside its own tools (``agents.py``).
+_ModelCallable: TypeAlias = "Tool | MultiStepAgent"
+
 _DEFAULT_IMAGE_MIME_TYPE = "image/png"
 _DATA_URL_PREFIX = "data:"
 
@@ -202,18 +210,23 @@ def to_output_message(output_message: ChatMessage) -> OutputMessage:
     return OutputMessage(role=role, parts=parts, finish_reason="")
 
 
-def _tool_parameters(tool: Tool) -> dict[str, Any] | None:
+def _tool_parameters(tool: _ModelCallable) -> dict[str, Any] | None:
     """Return the JSON Schema ``parameters`` object for a smolagents tool.
 
     A tool's ``inputs`` map is not a JSON Schema on its own: smolagents wraps it
     in an object schema, derives ``required`` from ``nullable``, and rewrites its
     non-JSON-Schema ``"any"`` type. ``get_tool_json_schema`` builds exactly the
     schema the provider receives.
+
+    ``get_tool_json_schema`` is annotated as taking a ``Tool``, but it reads
+    only ``name``, ``description`` and ``inputs``, and smolagents calls it with
+    managed agents too: ``ToolCallingAgent.tools_and_managed_agents`` feeds
+    ``tools_to_call_from`` (``agents.py``). Hence the cast.
     """
     try:
-        schema = get_tool_json_schema(tool)
+        schema = get_tool_json_schema(cast("Tool", tool))
         parameters = schema["function"]["parameters"]
-    except Exception:  # pylint: disable=broad-except
+    except BaseException:  # pylint: disable=broad-except
         _logger.debug(
             "Failed to build a JSON Schema for tool %s",
             tool.name,
@@ -224,21 +237,75 @@ def _tool_parameters(tool: Tool) -> dict[str, Any] | None:
 
 
 def to_tool_definitions(
-    tools: list[Tool] | None,
+    tools: Sequence[_ModelCallable] | None,
 ) -> list[ToolDefinition] | None:
     """Map smolagents tool objects to function tool definitions.
 
-    ``Tool.validate_arguments`` runs on every instantiation and requires a
-    non-empty ``name`` and a ``description``, so both are read directly.
+    ``Tool.validate_arguments`` requires every tool to have a non-empty
+    ``name`` and ``description``. A managed agent must also have both. An
+    invalid entry without a name is skipped instead of recorded under an empty
+    name.
     """
     if not tools:
         return None
-    definitions: list[ToolDefinition] = [
-        FunctionToolDefinition(
-            name=tool.name,
-            description=tool.description,
-            parameters=_tool_parameters(tool),
+    definitions: list[ToolDefinition] = []
+    for tool in tools:
+        name = tool.name
+        if not name:
+            continue
+        definitions.append(
+            FunctionToolDefinition(
+                name=name,
+                description=tool.description,
+                parameters=_tool_parameters(tool),
+            )
         )
-        for tool in tools
-    ]
-    return definitions
+    return definitions or None
+
+
+def _to_text(value: object) -> str:
+    """Stringify a value without running smolagents' side-effecting ``__str__``.
+
+    ``AgentText`` and ``AgentAudio`` subclass ``str`` but override ``__str__``
+    (``agent_types.py``), and ``AgentAudio.to_string()`` writes a ``.wav`` file
+    to a temp directory. Reading the underlying ``str`` avoids that.
+    """
+    if isinstance(value, str):
+        return str.__str__(value)
+    return str(value)
+
+
+def final_answer_parts(output: object) -> list[MessagePart]:
+    """Map an agent's final answer onto output message parts.
+
+    ``MultiStepAgent`` wraps the answer in ``handle_agent_output_types``, so an
+    image-producing run returns an ``AgentImage``, which subclasses a PIL image.
+    Recording it as a ``Blob`` keeps the content and matches how input images
+    are recorded; stringifying it would write the image to disk and record the
+    path instead.
+    """
+    if isinstance(output, Image):
+        if blob := _image_blob(output):
+            return [blob]
+        return []
+    return [Text(content=_to_text(output))]
+
+
+def task_to_input_messages(
+    task: str | None, images: list[Image | str] | None
+) -> list[InputMessage]:
+    """Build the agent-run input messages from the task string and images.
+
+    A call with neither a task nor images produces no message. ``run()``
+    without a task raises, so the failed call has no input to record.
+    """
+    parts: list[MessagePart] = []
+    if task:
+        parts.append(Text(content=task))
+    if isinstance(images, list):
+        for image in images:
+            if blob := _image_blob(image):
+                parts.append(blob)
+    if not parts:
+        return []
+    return [InputMessage(role="user", parts=parts)]