diff --git a/lib/crewai/src/crewai/llms/_finish_reason_utils.py b/lib/crewai/src/crewai/llms/_finish_reason_utils.py index e79befcc77..1462b44c84 100644 --- a/lib/crewai/src/crewai/llms/_finish_reason_utils.py +++ b/lib/crewai/src/crewai/llms/_finish_reason_utils.py @@ -11,13 +11,86 @@ from __future__ import annotations +import logging from typing import Any +# Providers spell "I ran out of output budget" differently: OpenAI/Azure Chat use +# ``length``, Anthropic ``max_tokens``, Bedrock ``max_tokens`` via ``stopReason``, +# Gemini ``MAX_TOKENS``, and the OpenAI Responses API reports +# ``incomplete_details.reason == "max_output_tokens"`` rather than a finish reason +# at all. Compared case-insensitively with separators stripped so one predicate +# covers all of them. +_TRUNCATION_REASONS = frozenset( + {"length", "maxtokens", "modellength", "maxoutputtokens"} +) + + def _as_str(value: Any) -> str | None: + """Return ``value`` when it is a string, else ``None``. + + Guards against provider types that are not plain strings (test doubles, + protobuf enums) leaking downstream. + """ return value if isinstance(value, str) else None +def is_truncated(finish_reason: str | None) -> bool: + """Whether ``finish_reason`` means the response was cut off by the token cap. + + Providers spell this differently: ``length`` (OpenAI, Azure), ``max_tokens`` + (Anthropic, Bedrock) and ``MAX_TOKENS`` (Gemini). Comparison is + case-insensitive with separators stripped so one predicate covers all of them. + + Args: + finish_reason: The raw finish/stop reason as reported by the provider. + + Returns: + ``True`` if the value denotes truncation by the output-token cap. + """ + if not isinstance(finish_reason, str): + return False + return ( + finish_reason.replace("_", "").replace("-", "").strip().lower() + in _TRUNCATION_REASONS + ) + + +def warn_if_truncated( + finish_reason: str | None, + max_tokens: int | float | None = None, + model: str | None = None, +) -> bool: + """Log a warning when a response was cut off by the token cap. + + A truncated response is otherwise indistinguishable from a complete one to + everything downstream, so the caller cannot tell "the model was interrupted" + from "the model answered badly". + + Args: + finish_reason: The raw finish/stop reason as reported by the provider. + max_tokens: The cap actually sent on the wire, so the message names the + setting the reader needs to change. Providers differ here + (``max_output_tokens`` on Gemini, ``max_completion_tokens`` on + OpenAI and Azure reasoning models), so callers pass their effective + value rather than a single shared attribute. + model: The model the request was sent to, for diagnostics. + + Returns: + ``True`` if a warning was emitted, ``False`` otherwise. + """ + if not is_truncated(finish_reason): + return False + logging.warning( + "Response truncated due to max_tokens limit (finish_reason=%r%s%s). " + "The output is incomplete; consider increasing max_tokens.", + finish_reason, + f", model={model}" if model else "", + f", max_tokens={max_tokens}" if max_tokens is not None else "", + ) + return True + + def extract_choices_finish_reason_and_id( response_or_chunk: Any, ) -> tuple[str | None, str | None]: diff --git a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py index 77236249af..a0dfbea52b 100644 --- a/lib/crewai/src/crewai/llms/providers/anthropic/completion.py +++ b/lib/crewai/src/crewai/llms/providers/anthropic/completion.py @@ -9,6 +9,7 @@ from crewai.events.types.llm_events import LLMCallType from crewai.hooks.dispatch import HookAborted +from crewai.llms._finish_reason_utils import warn_if_truncated from crewai.llms.base_llm import ( BaseLLM, JsonResponseFormat, @@ -1055,6 +1056,7 @@ def _handle_completion( self._track_token_usage_internal(usage) finish_reason, response_id = self._extract_finish_reason_and_id(response) + warn_if_truncated(finish_reason, self._effective_max_tokens(), self.model) if _is_pydantic_model_class(response_model) and response.content: if use_native_structured_output: @@ -1603,6 +1605,7 @@ async def _ahandle_completion( self._track_token_usage_internal(usage) finish_reason, response_id = self._extract_finish_reason_and_id(response) + warn_if_truncated(finish_reason, self._effective_max_tokens(), self.model) if _is_pydantic_model_class(response_model) and response.content: if use_native_structured_output: diff --git a/lib/crewai/src/crewai/llms/providers/azure/completion.py b/lib/crewai/src/crewai/llms/providers/azure/completion.py index 8cdb49cca7..a21611e8ac 100644 --- a/lib/crewai/src/crewai/llms/providers/azure/completion.py +++ b/lib/crewai/src/crewai/llms/providers/azure/completion.py @@ -10,7 +10,10 @@ from typing_extensions import Self from crewai.hooks.dispatch import HookAborted -from crewai.llms._finish_reason_utils import extract_choices_finish_reason_and_id +from crewai.llms._finish_reason_utils import ( + extract_choices_finish_reason_and_id, + warn_if_truncated, +) from crewai.llms.hooks.base import BaseInterceptor from crewai.utilities.agent_utils import is_context_length_exceeded from crewai.utilities.exceptions.context_window_exceeding_exception import ( @@ -873,6 +876,7 @@ def _process_completion_response( self._track_token_usage_internal(usage) finish_reason, response_id = self._extract_finish_reason_and_id(response) + warn_if_truncated(finish_reason, self._effective_max_tokens(), self.model) # Without available_functions, return tool_calls so the caller (executor) handles execution if message.tool_calls and not available_functions: diff --git a/lib/crewai/src/crewai/llms/providers/gemini/completion.py b/lib/crewai/src/crewai/llms/providers/gemini/completion.py index 98edc12383..2431593f01 100644 --- a/lib/crewai/src/crewai/llms/providers/gemini/completion.py +++ b/lib/crewai/src/crewai/llms/providers/gemini/completion.py @@ -11,6 +11,7 @@ from crewai.events.types.llm_events import LLMCallType from crewai.hooks.dispatch import HookAborted +from crewai.llms._finish_reason_utils import warn_if_truncated from crewai.llms.base_llm import BaseLLM, LLMCallBlockedError, llm_call_context from crewai.llms.hooks.base import BaseInterceptor from crewai.utilities.agent_utils import is_context_length_exceeded @@ -877,6 +878,7 @@ def _process_response_with_tools( Final response content or function call result """ finish_reason, response_id = self._extract_finish_reason_and_id(response) + warn_if_truncated(finish_reason, self._effective_max_tokens(), self.model) if response.candidates and (self.tools or available_functions): candidate = response.candidates[0] diff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.py index 20234e5019..4e5ae10f50 100644 --- a/lib/crewai/src/crewai/llms/providers/openai/completion.py +++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py @@ -37,7 +37,10 @@ from crewai.events.types.llm_events import LLMCallType from crewai.hooks.dispatch import HookAborted -from crewai.llms._finish_reason_utils import extract_choices_finish_reason_and_id +from crewai.llms._finish_reason_utils import ( + extract_choices_finish_reason_and_id, + warn_if_truncated, +) from crewai.llms.base_llm import ( BaseLLM, JsonResponseFormat, @@ -994,6 +997,13 @@ def _handle_responses( finish_reason, response_id = self._extract_responses_finish_reason_and_id( response ) + # The Responses API reports ``status`` ("incomplete"), not a finish + # reason, so the cause lives in ``incomplete_details.reason``. + warn_if_truncated( + self._responses_truncation_reason(response) or finish_reason, + self._effective_max_tokens(), + self.model, + ) if self.parse_tool_outputs: parsed_result = self._extract_builtin_tool_outputs(response) @@ -1141,6 +1151,13 @@ async def _ahandle_responses( finish_reason, response_id = self._extract_responses_finish_reason_and_id( response ) + # The Responses API reports ``status`` ("incomplete"), not a finish + # reason, so the cause lives in ``incomplete_details.reason``. + warn_if_truncated( + self._responses_truncation_reason(response) or finish_reason, + self._effective_max_tokens(), + self.model, + ) if self.parse_tool_outputs: parsed_result = self._extract_builtin_tool_outputs(response) @@ -1921,6 +1938,9 @@ def _handle_completion( ) parsed_object = parsed_response.choices[0].message.parsed if parsed_object: + warn_if_truncated( + parsed_finish_reason, self._effective_max_tokens(), self.model + ) self._emit_call_completed_event( response=parsed_object.model_dump_json(), call_type=LLMCallType.LLM_CALL, @@ -1946,6 +1966,7 @@ def _handle_completion( finish_reason, response_id = self._extract_chat_finish_reason_and_id( response ) + warn_if_truncated(finish_reason, self._effective_max_tokens(), self.model) # Without available_functions, return tool_calls so the caller (executor) handles execution if message.tool_calls and not available_functions: @@ -2349,6 +2370,9 @@ async def _ahandle_completion( ) parsed_object = parsed_response.choices[0].message.parsed if parsed_object: + warn_if_truncated( + parsed_finish_reason, self._effective_max_tokens(), self.model + ) self._emit_call_completed_event( response=parsed_object.model_dump_json(), call_type=LLMCallType.LLM_CALL, @@ -2374,6 +2398,7 @@ async def _ahandle_completion( finish_reason, response_id = self._extract_chat_finish_reason_and_id( response ) + warn_if_truncated(finish_reason, self._effective_max_tokens(), self.model) # Without available_functions, return tool_calls so the caller (executor) handles execution if message.tool_calls and not available_functions: @@ -2715,6 +2740,22 @@ def _extract_chat_finish_reason_and_id( """ return extract_choices_finish_reason_and_id(response) + @staticmethod + def _responses_truncation_reason(response: Any) -> str | None: + """Why a Responses-API call stopped early, when it did. + + ``status`` only says ``"incomplete"``; the cause is carried separately on + ``incomplete_details.reason`` (for example ``"max_output_tokens"``). + Returns ``None`` for complete responses and for shapes lacking the field. + """ + if getattr(response, "status", None) != "incomplete": + return None + details = getattr(response, "incomplete_details", None) + reason = getattr(details, "reason", None) + if reason is None and isinstance(details, dict): + reason = details.get("reason") + return reason if isinstance(reason, str) else None + @staticmethod def _extract_responses_finish_reason_and_id( response: Any, diff --git a/lib/crewai/tests/llms/test_finish_reason_utils.py b/lib/crewai/tests/llms/test_finish_reason_utils.py new file mode 100644 index 0000000000..bb3dec458d --- /dev/null +++ b/lib/crewai/tests/llms/test_finish_reason_utils.py @@ -0,0 +1,139 @@ +"""Truncation detection shared across providers. + +A response cut off by the token cap is otherwise indistinguishable from a complete +one, so callers cannot tell "the model was interrupted" from "the model answered +badly". These tests pin the behaviour rather than the wording. +""" + +import logging + +import pytest + +from crewai.llms._finish_reason_utils import is_truncated, warn_if_truncated + + +@pytest.mark.parametrize( + "finish_reason", + [ + "length", # OpenAI, Azure + "max_tokens", # Anthropic, Bedrock + "MAX_TOKENS", # Gemini + "maxTokens", + "max-tokens", + " Length ", + ], +) +def test_recognises_every_provider_spelling(finish_reason): + """Every vendor spells truncation differently and all of them must be caught. + + A provider whose spelling is missed looks identical to a complete response, + which is the failure this helper exists to prevent. + """ + assert is_truncated(finish_reason) is True + + +@pytest.mark.parametrize( + "finish_reason", + ["stop", "STOP", "tool_calls", "content_filter", "end_turn", "", None, 42, object()], +) +def test_ignores_non_truncation_reasons(finish_reason): + """Normal completions and non-string values must not raise a false alarm. + + A warning on every healthy turn is worse than no warning at all, because + readers learn to ignore it. + """ + assert is_truncated(finish_reason) is False + + +def test_warns_once_and_names_the_budget(caplog): + """One warning per truncated response, naming the cap that caused it. + + The cap has to appear or the reader cannot tell which setting to raise. + """ + with caplog.at_level(logging.WARNING): + emitted = warn_if_truncated("length", max_tokens=16, model="gpt-4o-mini") + + assert emitted is True + assert len(caplog.records) == 1 + message = caplog.records[0].getMessage() + # The reader needs to know which knob to turn, so the cap has to appear. + assert "16" in message + assert "gpt-4o-mini" in message + + +def test_silent_on_a_complete_response(caplog): + """A complete response must log nothing at all.""" + with caplog.at_level(logging.WARNING): + emitted = warn_if_truncated("stop", max_tokens=16, model="gpt-4o-mini") + + assert emitted is False + assert caplog.records == [] + + +def test_warns_without_optional_context(caplog): + """max_tokens is often unset, and a warning is still better than silence.""" + with caplog.at_level(logging.WARNING): + emitted = warn_if_truncated("max_tokens") + + assert emitted is True + assert len(caplog.records) == 1 + + +def test_warning_names_whichever_cap_the_provider_actually_sent(): + """Providers send different cap parameters, so the message must not assume one. + + Gemini sends ``max_output_tokens`` and OpenAI/Azure reasoning models send + ``max_completion_tokens``. Naming the wrong setting sends the reader to a + knob that will not change anything. + """ + import logging + + from crewai.llms._finish_reason_utils import warn_if_truncated + + for cap, model in [(2048, "gemini-2.5-flash"), (4096, "o3-mini"), (16, "claude-haiku-4-5")]: + records = [] + handler = logging.Handler() + handler.emit = records.append # type: ignore[method-assign] + logging.getLogger().addHandler(handler) + try: + assert warn_if_truncated("length", cap, model) is True + finally: + logging.getLogger().removeHandler(handler) + assert str(cap) in records[0].getMessage() + assert model in records[0].getMessage() + + +def test_recognises_the_responses_api_spelling(): + """The Responses API reports ``max_output_tokens``, not ``length``. + + That path exposes ``status`` rather than a finish reason, so the cause comes + from ``incomplete_details.reason``. Missing this spelling meant the warning + never fired on the Responses API at all. + """ + from crewai.llms._finish_reason_utils import is_truncated + + assert is_truncated("max_output_tokens") is True + assert is_truncated("maxOutputTokens") is True + assert is_truncated("content_filter") is False + + +def test_responses_truncation_reason_reads_incomplete_details(): + """Only an incomplete response yields a reason, and both shapes are accepted.""" + from types import SimpleNamespace + + from crewai.llms.providers.openai.completion import OpenAICompletion + + read = OpenAICompletion._responses_truncation_reason + + incomplete = SimpleNamespace( + status="incomplete", incomplete_details=SimpleNamespace(reason="max_output_tokens") + ) + assert read(incomplete) == "max_output_tokens" + + as_dict = SimpleNamespace( + status="incomplete", incomplete_details={"reason": "max_output_tokens"} + ) + assert read(as_dict) == "max_output_tokens" + + assert read(SimpleNamespace(status="completed", incomplete_details=None)) is None + assert read(SimpleNamespace(status="incomplete", incomplete_details=None)) is None