From 01ac97b755ea8e8470960834d69d3ea91701f2ee Mon Sep 17 00:00:00 2001 From: Joao Moura Date: Mon, 31 Aug 2026 23:27:41 -0700 Subject: [PATCH 1/2] fix(llms): send reasoning_effort to every openai reasoning model The completions path gated the parameter behind is_o1_model = "o1" in model.lower(), a literal substring test. gpt-5, o3 and o4-mini contain no "o1", so an explicitly configured effort was dropped and the model thought at the server default. The request still succeeded, so nothing surfaced -- one measured extraction ran 6.2s with the setting applied against 149.7s with it dropped. The gate could not be widened: is_o1_model also drives supports_function_calling, supports_stop_words and the system->user message rewrite, so marking gpt-5 as an o1 model would report that it cannot call tools. The parameter is forwarded unconditionally instead, matching the responses path, and a model that genuinely does not support it says so in a 400 that is retried once without the key. Also adds "minimal" to LLM.reasoning_effort, which gpt-5 accepts and the Literal omitted, so the cheapest setting was unreachable on the typed surface. Co-Authored-By: Claude Opus 5 (1M context) --- lib/crewai/src/crewai/llm.py | 2 +- .../llms/providers/openai/completion.py | 81 ++++- .../test_reasoning_effort_forwarding.py | 276 ++++++++++++++++++ 3 files changed, 352 insertions(+), 7 deletions(-) create mode 100644 lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py index 1eb52255fb..c5591d2ffb 100644 --- a/lib/crewai/src/crewai/llm.py +++ b/lib/crewai/src/crewai/llm.py @@ -386,7 +386,7 @@ class LLM(BaseLLM): api_base: str | None = None api_version: str | None = None callbacks: list[Any] | None = None - reasoning_effort: Literal["none", "low", "medium", "high"] | None = None + reasoning_effort: Literal["none", "minimal", "low", "medium", "high"] | None = None stream: bool = False interceptor: Any = None thinking: Any = None diff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.py index 20234e5019..898c26281e 100644 --- a/lib/crewai/src/crewai/llms/providers/openai/completion.py +++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py @@ -552,6 +552,16 @@ def dispatch(params: dict[str, Any]) -> str | Any: ) return dispatch(retry_params) + if self._rejects_reasoning_effort_as_unsupported(cause): + retry_params = self._without_reasoning_effort(completion_params) + if retry_params is not None: + logging.debug( + "Retrying %r without reasoning_effort: the model does " + "not support the parameter.", + self.model, + ) + return dispatch(retry_params) + if self.custom_openai or not self._is_responses_only_error(cause): raise self._remember_responses_only_model() @@ -683,6 +693,16 @@ async def dispatch(params: dict[str, Any]) -> str | Any: if retry_params is not None: return await dispatch(retry_params) + if self._rejects_reasoning_effort_as_unsupported(cause): + retry_params = self._without_reasoning_effort(completion_params) + if retry_params is not None: + logging.debug( + "Retrying %r without reasoning_effort: the model does " + "not support the parameter.", + self.model, + ) + return await dispatch(retry_params) + if self.custom_openai or not self._is_responses_only_error(cause): raise self._remember_responses_only_model() @@ -1765,6 +1785,49 @@ def _rejects_reasoning_effort_with_tools(error: BaseException) -> bool: message = str(source.get("message") or "").lower() return "function tools" in message and "reasoning_effort" in message + @staticmethod + def _rejects_reasoning_effort_as_unsupported(error: BaseException) -> bool: + """Whether a 400 is OpenAI refusing `reasoning_effort` for this model. + + Non-reasoning models reject the parameter itself, in one of two shapes: + + {"code": "unsupported_parameter", "param": "reasoning_effort", + "message": "Unsupported parameter: 'reasoning_effort' is not + supported with this model."} + + {"param": null, "message": "Unrecognized request argument + supplied: reasoning_effort"} + + Distinct from `_rejects_reasoning_effort_with_tools`: that is a + reasoning model refusing the parameter only alongside function tools, + and it recovers by sending "none" rather than by dropping the key. Also + deliberately does not match the "Unsupported value" 400 that o1/o3 + return for a bad *value* -- the model does support the parameter, so + silently dropping it would restore the very bug this recovers from. + """ + if not isinstance(error, BadRequestError): + return False + body = getattr(error, "body", None) + source = None + if isinstance(body, dict): + inner = body.get("error") + source = inner if isinstance(inner, dict) else body + if not isinstance(source, dict): + return False + message = str(source.get("message") or "").lower() + if "reasoning_effort" not in message: + return False + if source.get("code") == "unsupported_parameter": + return True + return "unrecognized request argument" in message + + @staticmethod + def _without_reasoning_effort(params: dict[str, Any]) -> dict[str, Any] | None: + """Params with `reasoning_effort` removed, or None if it was not set.""" + if "reasoning_effort" not in params: + return None + return {k: v for k, v in params.items() if k != "reasoning_effort"} + def _reasoning_effort_none_params( self, params: dict[str, Any] ) -> dict[str, Any] | None: @@ -1829,8 +1892,10 @@ def _prepare_completion_params( if self.top_logprobs is not None: params["top_logprobs"] = self.top_logprobs - # Handle o1 model specific parameters - if self.is_o1_model and self.reasoning_effort: + # Forwarded for every model, not just o1: gpt-5, o3 and o4-mini are all + # reasoning models whose names contain no "o1". A model that does not + # support the parameter rejects it, and the caller retries without it. + if self.reasoning_effort: params["reasoning_effort"] = self.reasoning_effort if self.response_format is not None: @@ -2043,9 +2108,11 @@ def _handle_completion( logging.error(f"Context window exceeded: {e}") raise LLMContextLengthExceededError(str(e)) from e - # `_call_completions` retries this one, so reporting a failed call + # `_call_completions` retries these, so reporting a failed call # here would surface an error the caller never experiences. - if self._rejects_reasoning_effort_with_tools(e): + if self._rejects_reasoning_effort_with_tools( + e + ) or self._rejects_reasoning_effort_as_unsupported(e): raise error_msg = f"OpenAI API call failed: {e!s}" @@ -2471,9 +2538,11 @@ async def _ahandle_completion( logging.error(f"Context window exceeded: {e}") raise LLMContextLengthExceededError(str(e)) from e - # `_call_completions` retries this one, so reporting a failed call + # `_call_completions` retries these, so reporting a failed call # here would surface an error the caller never experiences. - if self._rejects_reasoning_effort_with_tools(e): + if self._rejects_reasoning_effort_with_tools( + e + ) or self._rejects_reasoning_effort_as_unsupported(e): raise error_msg = f"OpenAI API call failed: {e!s}" diff --git a/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py b/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py new file mode 100644 index 0000000000..3123cbf6ee --- /dev/null +++ b/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py @@ -0,0 +1,276 @@ +"""`reasoning_effort` reaches every reasoning model, not just o1. + +The completions path used to gate the parameter behind +``is_o1_model = "o1" in model.lower()``, a literal substring test. gpt-5, o3 and +o4-mini contain no "o1", so an explicitly configured effort was dropped and the +model thought at the server default -- silently, since the request still +succeeded. + +The gate could not simply be widened: ``is_o1_model`` also drives +``supports_function_calling``, ``supports_stop_words`` and the system->user +message rewrite, so marking gpt-5 as an o1 model would report that it cannot +call tools. The parameter is forwarded unconditionally instead, and a model that +genuinely does not support it says so in a 400 that is retried without the key. +""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest +from openai import BadRequestError + +from crewai.llm import LLM +from crewai.llms.providers.openai.completion import OpenAICompletion + + +MESSAGES = [{"role": "user", "content": "hi"}] + +REASONING_MODELS = ["gpt-5", "gpt-5-mini", "o3", "o3-mini", "o4-mini", "o1"] + + +def build(model: str = "gpt-5", **kwargs: Any) -> OpenAICompletion: + return OpenAICompletion(model=model, api_key="sk-test", **kwargs) + + +def _bad_request(message: str, **source: Any) -> BadRequestError: + body = {"error": {"message": message, "type": "invalid_request_error", **source}} + return BadRequestError( + message, + response=httpx.Response( + 400, + json=body, + request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"), + ), + body=body, + ) + + +def unsupported_parameter_error() -> BadRequestError: + """What a non-reasoning model returns for the parameter itself.""" + return _bad_request( + "Unsupported parameter: 'reasoning_effort' is not supported with this model.", + param="reasoning_effort", + code="unsupported_parameter", + ) + + +def unrecognized_argument_error() -> BadRequestError: + """The other shape, where `param` is null.""" + return _bad_request( + "Unrecognized request argument supplied: reasoning_effort", param=None + ) + + +def unsupported_value_error() -> BadRequestError: + """A bad *value* -- the model does support the parameter.""" + return _bad_request( + "Unsupported value: 'reasoning_effort' does not support 'none'.", + param="reasoning_effort", + ) + + +class TestParameterReachesTheModel: + @pytest.mark.parametrize("model", REASONING_MODELS) + def test_forwarded_for_every_reasoning_model(self, model): + params = build(model, reasoning_effort="high")._prepare_completion_params( + MESSAGES + ) + + assert params["reasoning_effort"] == "high" + + def test_minimal_is_forwarded(self): + """gpt-5's cheapest setting, and the one extraction workloads want.""" + params = build("gpt-5", reasoning_effort="minimal")._prepare_completion_params( + MESSAGES + ) + + assert params["reasoning_effort"] == "minimal" + + @pytest.mark.parametrize("model", ["gpt-4o", "gpt-5", "o3"]) + def test_absent_when_not_configured(self, model): + """Unset must stay off the wire, whatever the model.""" + assert "reasoning_effort" not in build(model)._prepare_completion_params( + MESSAGES + ) + + +class TestO1FlagUntouched: + """The gate was shared; widening it would have broken these.""" + + @pytest.mark.parametrize("model", ["gpt-5", "o3", "o4-mini"]) + def test_reasoning_models_still_support_tools(self, model): + assert build(model).supports_function_calling() is True + + @pytest.mark.parametrize("model", ["gpt-5", "o3"]) + def test_is_o1_model_still_only_matches_o1(self, model): + assert build(model).is_o1_model is False + assert build("o1").is_o1_model is True + + def test_system_messages_are_not_rewritten_for_gpt5(self): + formatted = build("gpt-5")._format_messages( + [{"role": "system", "content": "be terse"}] + ) + + assert formatted[0]["role"] == "system" + + +class TestErrorDetection: + def test_matches_unsupported_parameter(self): + assert OpenAICompletion._rejects_reasoning_effort_as_unsupported( + unsupported_parameter_error() + ) + + def test_matches_unrecognized_argument(self): + assert OpenAICompletion._rejects_reasoning_effort_as_unsupported( + unrecognized_argument_error() + ) + + def test_ignores_an_unsupported_value(self): + """Dropping the key here would silently restore the original bug.""" + assert not OpenAICompletion._rejects_reasoning_effort_as_unsupported( + unsupported_value_error() + ) + + def test_ignores_a_400_about_another_parameter(self): + assert not OpenAICompletion._rejects_reasoning_effort_as_unsupported( + _bad_request( + "Unsupported parameter: 'temperature' is not supported.", + param="temperature", + code="unsupported_parameter", + ) + ) + + def test_ignores_unrelated_exceptions(self): + assert not OpenAICompletion._rejects_reasoning_effort_as_unsupported( + RuntimeError("boom") + ) + + +class TestRetryParams: + def test_removes_the_key(self): + params = OpenAICompletion._without_reasoning_effort( + {"model": "gpt-4o", "reasoning_effort": "high"} + ) + + assert params == {"model": "gpt-4o"} + + def test_returns_none_when_absent(self): + """Nothing left to drop -- the retry must not loop.""" + assert OpenAICompletion._without_reasoning_effort({"model": "gpt-4o"}) is None + + +class TestRetryBehaviour: + def test_retries_without_the_key_and_succeeds(self, monkeypatch): + llm = build("gpt-4o", reasoning_effort="high") + seen: list[dict] = [] + + def fake_handle(params, **kwargs): + seen.append(params) + if "reasoning_effort" in params: + raise unsupported_parameter_error() + return "ok" + + monkeypatch.setattr(llm, "_handle_completion", fake_handle) + + assert llm._call_completions(MESSAGES) == "ok" + assert len(seen) == 2, "expected one rejected call and one retry" + assert seen[0]["reasoning_effort"] == "high" + assert "reasoning_effort" not in seen[1] + + @pytest.mark.asyncio + async def test_retries_on_the_async_path(self, monkeypatch): + llm = build("gpt-4o", reasoning_effort="high") + seen: list[dict] = [] + + async def fake_handle(params, **kwargs): + seen.append(params) + if "reasoning_effort" in params: + raise unsupported_parameter_error() + return "ok" + + monkeypatch.setattr(llm, "_ahandle_completion", fake_handle) + + assert await llm._acall_completions(MESSAGES) == "ok" + assert len(seen) == 2 + assert "reasoning_effort" not in seen[1] + + def test_retries_on_the_streaming_path(self, monkeypatch): + llm = build("gpt-4o", reasoning_effort="high", stream=True) + seen: list[dict] = [] + + def fake_handle(params, **kwargs): + seen.append(params) + if "reasoning_effort" in params: + raise unsupported_parameter_error() + return "ok" + + monkeypatch.setattr(llm, "_handle_streaming_completion", fake_handle) + + assert llm._call_completions(MESSAGES) == "ok" + assert len(seen) == 2 + assert "reasoning_effort" not in seen[1] + + @pytest.mark.asyncio + async def test_retries_on_the_async_streaming_path(self, monkeypatch): + llm = build("gpt-4o", reasoning_effort="high", stream=True) + seen: list[dict] = [] + + async def fake_handle(params, **kwargs): + seen.append(params) + if "reasoning_effort" in params: + raise unsupported_parameter_error() + return "ok" + + monkeypatch.setattr(llm, "_ahandle_streaming_completion", fake_handle) + + assert await llm._acall_completions(MESSAGES) == "ok" + assert len(seen) == 2 + assert "reasoning_effort" not in seen[1] + + def test_does_not_retry_forever(self, monkeypatch): + llm = build("gpt-4o", reasoning_effort="high") + calls: list[dict] = [] + + def always_fail(params, **kwargs): + calls.append(params) + raise unsupported_parameter_error() + + monkeypatch.setattr(llm, "_handle_completion", always_fail) + + with pytest.raises(BadRequestError, match="not supported with this model"): + llm._call_completions(MESSAGES) + + assert len(calls) == 2, "one original call plus exactly one retry" + + def test_an_unsupported_value_surfaces(self, monkeypatch): + """A bad value is the caller's mistake and must not be papered over.""" + llm = build("o3", reasoning_effort="none") + calls: list[dict] = [] + + def always_fail(params, **kwargs): + calls.append(params) + raise unsupported_value_error() + + monkeypatch.setattr(llm, "_handle_completion", always_fail) + + with pytest.raises(BadRequestError, match="does not support 'none'"): + llm._call_completions(MESSAGES) + + assert len(calls) == 1, "a bad value must not be retried" + + +class TestLLMSurface: + @pytest.mark.parametrize("effort", ["none", "minimal", "low", "medium", "high"]) + def test_llm_accepts_every_documented_effort(self, effort): + llm = LLM(model="gpt-5", reasoning_effort=effort, is_litellm=True) + + assert llm.reasoning_effort == effort + + def test_reaches_the_wire_through_the_llm_factory(self): + """End to end: LLM(...) -> native provider -> request params.""" + llm = LLM(model="gpt-5", reasoning_effort="minimal", is_litellm=False) + params = llm._prepare_completion_params(MESSAGES) + + assert params["reasoning_effort"] == "minimal" From 2a161984ef23ffb70365abab2ca73032e7bcc3df Mon Sep 17 00:00:00 2001 From: Joao Moura Date: Mon, 31 Aug 2026 23:39:10 -0700 Subject: [PATCH 2/2] refactor(llms): gate reasoning_effort on model shape, not every model Forwarding to every model made a non-reasoning model pay a rejected request and a retry on every call. `_supports_reasoning_effort` matches on shape instead -- the o-series, and GPT generation 5 onwards -- so gpt-4o and gpt-4.1 never send the parameter at all. Matched by shape rather than by a list of names so a new member of an existing family works without a release here; gpt-6 and o5 already classify correctly. The unsupported-parameter retry stays as a safety net for the case the shape match is wrong for a future family, where it costs nothing when the match is right. Co-Authored-By: Claude Opus 5 (1M context) --- .../llms/providers/openai/completion.py | 33 ++++++-- .../test_reasoning_effort_forwarding.py | 75 +++++++++++++++++-- 2 files changed, 95 insertions(+), 13 deletions(-) diff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.py index 898c26281e..a59d3ff69e 100644 --- a/lib/crewai/src/crewai/llms/providers/openai/completion.py +++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py @@ -5,7 +5,8 @@ import json import logging import os -from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict +import re +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypedDict import httpx from openai import ( @@ -70,6 +71,27 @@ # per process rather than on every call. _LEARNED_RESPONSES_ONLY_MODELS: set[str] = set() +# `reasoning_effort` is accepted by the o-series and by GPT generation 5 onwards. +# Matched by shape rather than by a list of names so a new member of an existing +# family works without a release here; `gpt-4o` and `gpt-4.1` parse to generation +# 4 and are excluded. An over-match is recovered rather than fatal -- the call is +# retried without the parameter when the API rejects it. +_O_SERIES_MODEL = re.compile(r"^o\d") +_GPT_GENERATION = re.compile(r"^gpt-(\d+)") +_MIN_REASONING_GPT_GENERATION: Final[int] = 5 + + +def _supports_reasoning_effort(model: str) -> bool: + """Whether the model accepts `reasoning_effort` on /v1/chat/completions.""" + name = model.rsplit("/", 1)[-1].lower() + if _O_SERIES_MODEL.match(name): + return True + generation = _GPT_GENERATION.match(name) + return ( + generation is not None + and int(generation.group(1)) >= _MIN_REASONING_GPT_GENERATION + ) + class WebSearchResult(TypedDict, total=False): """Result from web search built-in tool.""" @@ -1892,10 +1914,11 @@ def _prepare_completion_params( if self.top_logprobs is not None: params["top_logprobs"] = self.top_logprobs - # Forwarded for every model, not just o1: gpt-5, o3 and o4-mini are all - # reasoning models whose names contain no "o1". A model that does not - # support the parameter rejects it, and the caller retries without it. - if self.reasoning_effort: + # Not gated on `is_o1_model`: that is a literal "o1" substring test, so + # gpt-5, o3 and o4-mini failed it and silently thought at the server + # default. It also drives tool support and message rewriting, so it + # cannot be widened to mean "is a reasoning model". + if self.reasoning_effort and _supports_reasoning_effort(self.model): params["reasoning_effort"] = self.reasoning_effort if self.response_format is not None: diff --git a/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py b/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py index 3123cbf6ee..e232572393 100644 --- a/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py +++ b/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py @@ -9,8 +9,11 @@ The gate could not simply be widened: ``is_o1_model`` also drives ``supports_function_calling``, ``supports_stop_words`` and the system->user message rewrite, so marking gpt-5 as an o1 model would report that it cannot -call tools. The parameter is forwarded unconditionally instead, and a model that -genuinely does not support it says so in a 400 that is retried without the key. +call tools. ``_supports_reasoning_effort`` is a separate predicate matched on +model *shape* -- the o-series, and GPT generation 5 onwards -- so a new member of +an existing family needs no release, and a non-reasoning model never pays a +wasted round trip. If the shape match is ever wrong for a future family, the +400 is retried without the key rather than surfacing. """ from __future__ import annotations @@ -22,7 +25,10 @@ from openai import BadRequestError from crewai.llm import LLM -from crewai.llms.providers.openai.completion import OpenAICompletion +from crewai.llms.providers.openai.completion import ( + OpenAICompletion, + _supports_reasoning_effort, +) MESSAGES = [{"role": "user", "content": "hi"}] @@ -95,6 +101,59 @@ def test_absent_when_not_configured(self, model): MESSAGES ) + @pytest.mark.parametrize("model", ["gpt-4o", "gpt-4.1", "gpt-3.5-turbo"]) + def test_not_sent_to_non_reasoning_models(self, model): + """No wasted round trip: these never reach the API with the parameter.""" + params = build(model, reasoning_effort="high")._prepare_completion_params( + MESSAGES + ) + + assert "reasoning_effort" not in params + + +class TestSupportedModelShape: + """Matched on shape so a new family member needs no release.""" + + @pytest.mark.parametrize( + "model", + [ + "o1", + "o1-mini", + "o1-preview", + "o3", + "o3-mini", + "o4-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5.1", + "gpt-5.6-sol", + "openai/gpt-5", + ], + ) + def test_supported(self, model): + assert _supports_reasoning_effort(model) is True + + @pytest.mark.parametrize( + "model", + [ + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-4-turbo", + "gpt-3.5-turbo", + "chatgpt-4o-latest", + "omni-moderation-latest", + "text-embedding-3-small", + ], + ) + def test_unsupported(self, model): + assert _supports_reasoning_effort(model) is False + + @pytest.mark.parametrize("model", ["gpt-6", "gpt-7-turbo", "o5", "o9-mini"]) + def test_future_family_members_need_no_release(self, model): + """The staleness this replaced: an unreleased family must still match.""" + assert _supports_reasoning_effort(model) is True + class TestO1FlagUntouched: """The gate was shared; widening it would have broken these.""" @@ -163,7 +222,7 @@ def test_returns_none_when_absent(self): class TestRetryBehaviour: def test_retries_without_the_key_and_succeeds(self, monkeypatch): - llm = build("gpt-4o", reasoning_effort="high") + llm = build("gpt-6-future", reasoning_effort="high") seen: list[dict] = [] def fake_handle(params, **kwargs): @@ -181,7 +240,7 @@ def fake_handle(params, **kwargs): @pytest.mark.asyncio async def test_retries_on_the_async_path(self, monkeypatch): - llm = build("gpt-4o", reasoning_effort="high") + llm = build("gpt-6-future", reasoning_effort="high") seen: list[dict] = [] async def fake_handle(params, **kwargs): @@ -197,7 +256,7 @@ async def fake_handle(params, **kwargs): assert "reasoning_effort" not in seen[1] def test_retries_on_the_streaming_path(self, monkeypatch): - llm = build("gpt-4o", reasoning_effort="high", stream=True) + llm = build("gpt-6-future", reasoning_effort="high", stream=True) seen: list[dict] = [] def fake_handle(params, **kwargs): @@ -214,7 +273,7 @@ def fake_handle(params, **kwargs): @pytest.mark.asyncio async def test_retries_on_the_async_streaming_path(self, monkeypatch): - llm = build("gpt-4o", reasoning_effort="high", stream=True) + llm = build("gpt-6-future", reasoning_effort="high", stream=True) seen: list[dict] = [] async def fake_handle(params, **kwargs): @@ -230,7 +289,7 @@ async def fake_handle(params, **kwargs): assert "reasoning_effort" not in seen[1] def test_does_not_retry_forever(self, monkeypatch): - llm = build("gpt-4o", reasoning_effort="high") + llm = build("gpt-6-future", reasoning_effort="high") calls: list[dict] = [] def always_fail(params, **kwargs):