fix(voice/realtime): add tool_reply_after_audio to prevent duplicate speech

livekit-agents/livekit/agents/voice/agent_activity.py

@@ -3853,11 +3853,16 @@ def _create_assistant_message(
         if len(tool_output.output) > 0:
             speech_handle._num_steps += 1
 
+            response_had_audio = started_speaking_at is not None and any(
+                entry.out.played != "skipped" for entry in message_outputs
+            )
+
             new_fnc_outputs: list[llm.FunctionCallOutput] = []
             generate_tool_reply: bool = False
             fnc_executed_ev = FunctionToolsExecutedEvent(
                 function_calls=[], function_call_outputs=[]
             )
+            fnc_executed_ev._response_had_audio = response_had_audio
             new_agent_task: Agent | None = None
             ignore_task_switch = False
 
@@ -3950,9 +3955,11 @@ async def _wait_for_auto_tool_reply() -> None:
                             self._pending_auto_tool_reply_fut = None
                         auto_reply_fut.set_result(None)
 
+            skip_for_audio = response_had_audio and self._session._tool_reply_after_audio == "skip"
             if (
                 fnc_executed_ev._reply_required
                 and not self._rt_session.capabilities.auto_tool_reply_generation
+                and not skip_for_audio
             ):
                 self._rt_session.interrupt()
 

livekit-agents/livekit/agents/voice/agent_session.py

@@ -474,6 +474,11 @@ def __init__(
         self._async_tool_options = _resolve_async_tool_options(
             tool_handling.get("async_options") if is_given(tool_handling) else None
         )
+        self._tool_reply_after_audio: Literal["always", "skip"] = (
+            tool_handling.get("tool_reply_after_audio", "always")
+            if is_given(tool_handling)
+            else "always"
+        )
 
         # unrecoverable error counts, reset after agent speaking
         self._llm_error_counts = 0

livekit-agents/livekit/agents/voice/events.py

@@ -430,6 +430,7 @@ class FunctionToolsExecutedEvent(BaseModel):
     created_at: float = Field(default_factory=time.time)
     _reply_required: bool = PrivateAttr(default=False)
     _handoff_required: bool = PrivateAttr(default=False)
+    _response_had_audio: bool = PrivateAttr(default=False)
 
     def zipped(self) -> list[tuple[FunctionCall, FunctionCallOutput | None]]:
         """Return calls paired with outputs by list position."""
@@ -449,6 +450,13 @@ def has_tool_reply(self) -> bool:
     def has_agent_handoff(self) -> bool:
         return self._handoff_required
 
+    @property
+    def response_had_audio(self) -> bool:
+        """Whether the response that triggered these function calls also produced
+        audible audio output. Useful for deciding whether a post-tool reply would
+        duplicate speech that was already delivered to the user."""
+        return self._response_had_audio
+
     @model_validator(mode="after")
     def verify_lists_length(self) -> Self:
         if len(self.function_calls) != len(self.function_call_outputs):

livekit-agents/livekit/agents/voice/tool_executor.py

@@ -120,6 +120,7 @@ class ToolHandlingOptions(TypedDict, total=False):
         AgentSession(
             tool_handling={
                 "async_options": {"update_template": "..."},
+                "tool_reply_after_audio": "skip",
             },
         )
 
@@ -130,6 +131,16 @@ class ToolHandlingOptions(TypedDict, total=False):
     """Templates injected around async tool dispatch (``ctx.update()``, duplicate
     handling, coalesced replies). Unmentioned keys keep their defaults."""
 
+    tool_reply_after_audio: Literal["always", "skip"]
+    """Controls post-tool reply generation when the realtime response that
+    triggered the function call also produced audible audio output.
+
+    - ``"always"`` (default): always generate a reply after tool execution,
+      even if the response already produced audio.
+    - ``"skip"``: skip the post-tool reply when the triggering response
+      already delivered audio to the user, preventing duplicate speech.
+    """
+
 
 def _render(template: str | Callable[[Any], str], args: dict[str, Any]) -> str:
     """Render a template: callables receive ``args``; strings use ``str.format(**args)``."""

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/llm.py

@@ -190,7 +190,7 @@ def provider(self) -> str:
     @staticmethod
     def with_azure(
         *,
-        model: str | ChatModels = "gpt-4o",
+        model: str | ChatModels | None = None,
         azure_endpoint: str | None = None,
         azure_deployment: str | None = None,
         api_version: str | None = None,
@@ -239,7 +239,7 @@ def with_azure(
         )  # type: ignore
 
         llm = LLM(
-            model=model,
+            model=model if model is not None else (azure_deployment or ""),
             client=azure_client,
             user=user,
             temperature=temperature,

tests/test_tool_reply_after_audio.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import pytest
+
+from livekit.agents import AgentSession
+from livekit.agents.voice.events import FunctionToolsExecutedEvent
+from livekit.agents.voice.tool_executor import ToolHandlingOptions
+
+pytestmark = pytest.mark.unit
+
+
+class TestFunctionToolsExecutedEventResponseHadAudio:
+    def test_default_response_had_audio_is_false(self) -> None:
+        ev = FunctionToolsExecutedEvent(
+            function_calls=[],
+            function_call_outputs=[],
+        )
+        assert ev.response_had_audio is False
+
+    def test_response_had_audio_set_true(self) -> None:
+        ev = FunctionToolsExecutedEvent(
+            function_calls=[],
+            function_call_outputs=[],
+        )
+        ev._response_had_audio = True
+        assert ev.response_had_audio is True
+
+    def test_cancel_tool_reply_still_works(self) -> None:
+        ev = FunctionToolsExecutedEvent(
+            function_calls=[],
+            function_call_outputs=[],
+        )
+        ev._reply_required = True
+        ev._response_had_audio = True
+        assert ev.has_tool_reply is True
+        ev.cancel_tool_reply()
+        assert ev.has_tool_reply is False
+
+
+class TestToolHandlingOptionsConfig:
+    def test_default_tool_reply_after_audio(self) -> None:
+        opts: ToolHandlingOptions = {}
+        assert opts.get("tool_reply_after_audio", "always") == "always"
+
+    def test_skip_tool_reply_after_audio(self) -> None:
+        opts: ToolHandlingOptions = {"tool_reply_after_audio": "skip"}
+        assert opts["tool_reply_after_audio"] == "skip"
+
+    def test_always_tool_reply_after_audio(self) -> None:
+        opts: ToolHandlingOptions = {"tool_reply_after_audio": "always"}
+        assert opts["tool_reply_after_audio"] == "always"
+
+
+class TestAgentSessionToolReplyConfig:
+    def test_session_defaults_to_always(self) -> None:
+        session = AgentSession()
+        assert session._tool_reply_after_audio == "always"
+
+    def test_session_accepts_skip(self) -> None:
+        session = AgentSession(tool_handling={"tool_reply_after_audio": "skip"})
+        assert session._tool_reply_after_audio == "skip"
+
+    def test_session_accepts_always_explicit(self) -> None:
+        session = AgentSession(tool_handling={"tool_reply_after_audio": "always"})
+        assert session._tool_reply_after_audio == "always"
+
+    def test_session_with_empty_tool_handling(self) -> None:
+        session = AgentSession(tool_handling={})
+        assert session._tool_reply_after_audio == "always"